xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into feature/vnode

This commit is contained in:
Hongze Cheng 2021-12-28 20:40:16 +08:00
parent 0c0525860b 64164ea46b
commit 311b044e02
91 changed files with 4661 additions and 4430 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
include/client/taos.h
View File

@ -193,6 +193,9 @@ DLL_EXPORT void taos_close_stream(TAOS_STREAM *tstr);
DLL_EXPORT int taos_load_table_info(TAOS *taos, const char* tableNameList);
DLL_EXPORT TAOS_RES* taos_schemaless_insert(TAOS* taos, char* lines[], int numLines, int protocol, int precision);
DLL_EXPORT TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen);
#ifdef __cplusplus
}
#endif

12
include/common/taosdef.h
View File

@ -38,12 +38,12 @@ typedef enum {
} EQType;
typedef enum {
TSDB_SUPER_TABLE = 1, // super table
TSDB_CHILD_TABLE = 2, // table created from super table
TSDB_NORMAL_TABLE = 3, // ordinary table
TSDB_STREAM_TABLE = 4, // table created from stream computing
TSDB_TEMP_TABLE = 5, // temp table created by nest query
TSDB_TABLE_MAX = 6
TSDB_SUPER_TABLE = 1, // super table
TSDB_CHILD_TABLE = 2, // table created from super table
TSDB_NORMAL_TABLE = 3, // ordinary table
TSDB_STREAM_TABLE = 4, // table created from stream computing
TSDB_TEMP_TABLE = 5, // temp table created by nest query
TSDB_TABLE_MAX = 6
} ETableType;
typedef enum {

1
include/common/tep.h
View File

@ -13,5 +13,6 @@ int taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port);
bool isEpsetEqual(const SEpSet *s1, const SEpSet *s2);
void updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet);
SEpSet getEpSet_s(SCorEpSet *pEpSet);
#endif // TDENGINE_TEP_H

71
include/common/tmsg.h
View File

@ -219,26 +219,6 @@ typedef struct {
char data[];
} SMDCreateTableMsg;
// typedef struct {
// int32_t len; // one create table message
// char tableName[TSDB_TABLE_FNAME_LEN];
// int16_t numOfColumns;
// int16_t sqlLen; // the length of SQL, it starts after schema , sql is a null-terminated string
// int8_t igExists;
// int8_t rspMeta;
// int8_t reserved[16];
// char schema[];
//} SCreateTableMsg;
typedef struct {
char tableName[TSDB_TABLE_FNAME_LEN];
int16_t numOfColumns;
int16_t numOfTags;
int8_t igExists;
int8_t rspMeta;
char schema[];
} SCreateCTableMsg;
typedef struct {
char name[TSDB_TABLE_FNAME_LEN];
int8_t igExists;
@ -372,7 +352,7 @@ typedef struct SColIndex {
int16_t colId; // column id
int16_t colIndex; // column index in colList if it is a normal column or index in tagColList if a tag
int16_t flag; // denote if it is a tag or a normal column
char name[TSDB_COL_NAME_LEN + TSDB_DB_NAME_LEN + 1];
char name[TSDB_DB_FNAME_LEN];
} SColIndex;
typedef struct SColumnFilterInfo {
@ -519,7 +499,7 @@ typedef struct SRetrieveTableRsp {
} SRetrieveTableRsp;
typedef struct {
char db[TSDB_FULL_DB_NAME_LEN];
char db[TSDB_DB_FNAME_LEN];
int32_t numOfVgroups;
int32_t cacheBlockSize; // MB
int32_t totalBlocks;
@ -543,7 +523,7 @@ typedef struct {
} SCreateDbMsg;
typedef struct {
char db[TSDB_FULL_DB_NAME_LEN];
char db[TSDB_DB_FNAME_LEN];
int32_t totalBlocks;
int32_t daysToKeep0;
int32_t daysToKeep1;
@ -693,7 +673,7 @@ typedef struct {
typedef struct {
int32_t vgId;
int32_t dnodeId;
char db[TSDB_FULL_DB_NAME_LEN];
char db[TSDB_DB_FNAME_LEN];
uint64_t dbUid;
int32_t vgVersion;
int32_t cacheBlockSize;
@ -720,7 +700,7 @@ typedef struct {
typedef struct {
int32_t vgId;
int32_t dnodeId;
char db[TSDB_FULL_DB_NAME_LEN];
char db[TSDB_DB_FNAME_LEN];
uint64_t dbUid;
} SDropVnodeMsg, SSyncVnodeMsg, SCompactVnodeMsg;
@ -796,7 +776,7 @@ typedef struct {
} STagData;
typedef struct {
char db[TSDB_FULL_DB_NAME_LEN];
char db[TSDB_DB_FNAME_LEN];
int32_t vgVersion;
int32_t vgNum;
int8_t hashMethod;
@ -810,13 +790,13 @@ typedef struct {
*/
typedef struct {
int8_t type;
char db[TSDB_FULL_DB_NAME_LEN];
char db[TSDB_DB_FNAME_LEN];
int16_t payloadLen;
char payload[];
} SShowMsg;
typedef struct {
char db[TSDB_FULL_DB_NAME_LEN];
char db[TSDB_DB_FNAME_LEN];
int32_t numOfVgroup;
int32_t vgid[];
} SCompactMsg;
@ -827,7 +807,7 @@ typedef struct SShowRsp {
} SShowRsp;
typedef struct {
char ep[TSDB_FQDN_LEN]; // end point, hostname:port
char fqdn[TSDB_FQDN_LEN]; // end point, hostname:port
int32_t port;
} SCreateDnodeMsg;
@ -1019,7 +999,7 @@ typedef struct {
} SUpdateTagValRsp;
typedef struct SSubQueryMsg {
uint64_t schedulerId;
uint64_t sId;
uint64_t queryId;
uint64_t taskId;
uint32_t contentLen;
@ -1027,7 +1007,7 @@ typedef struct SSubQueryMsg {
} SSubQueryMsg;
typedef struct SResReadyMsg {
uint64_t schedulerId;
uint64_t sId;
uint64_t queryId;
uint64_t taskId;
} SResReadyMsg;
@ -1037,13 +1017,13 @@ typedef struct SResReadyRsp {
} SResReadyRsp;
typedef struct SResFetchMsg {
uint64_t schedulerId;
uint64_t sId;
uint64_t queryId;
uint64_t taskId;
} SResFetchMsg;
typedef struct SSchTasksStatusMsg {
uint64_t schedulerId;
uint64_t sId;
} SSchTasksStatusMsg;
typedef struct STaskStatus {
@ -1058,7 +1038,7 @@ typedef struct SSchedulerStatusRsp {
} SSchedulerStatusRsp;
typedef struct STaskCancelMsg {
uint64_t schedulerId;
uint64_t sId;
uint64_t queryId;
uint64_t taskId;
} STaskCancelMsg;
@ -1068,7 +1048,7 @@ typedef struct STaskCancelRsp {
} STaskCancelRsp;
typedef struct STaskDropMsg {
uint64_t schedulerId;
uint64_t sId;
uint64_t queryId;
uint64_t taskId;
} STaskDropMsg;
@ -1077,6 +1057,27 @@ typedef struct STaskDropRsp {
int32_t code;
} STaskDropRsp;
typedef struct {
int8_t igExists;
char* name;
char* phyPlan;
} SCMCreateTopicReq;
static FORCE_INLINE int tSerializeSCMCreateTopicReq(void** buf, const SCMCreateTopicReq* pReq) {
int tlen = 0;
tlen += taosEncodeString(buf, pReq->name);
tlen += taosEncodeFixedI8(buf, pReq->igExists);
tlen += taosEncodeString(buf, pReq->phyPlan);
return tlen;
}
static FORCE_INLINE void* tDeserializeSCMCreateTopicReq(void* buf, SCMCreateTopicReq* pReq) {
buf = taosDecodeFixedI8(buf, &(pReq->igExists));
buf = taosDecodeString(buf, &(pReq->name));
buf = taosDecodeString(buf, &(pReq->phyPlan));
return buf;
}
typedef struct {
char name[TSDB_TOPIC_FNAME_LEN];
int8_t igExists;

388
include/common/ttokendef.h
View File

@ -13,205 +13,199 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_COMMON_TOKEN_DEF_H_
#define _TD_COMMON_TOKEN_DEF_H_
#ifndef TDENGINE_TTOKENDEF_H
#define TDENGINE_TTOKENDEF_H
#define TK_ID 1
#define TK_BOOL 2
#define TK_TINYINT 3
#define TK_SMALLINT 4
#define TK_INTEGER 5
#define TK_BIGINT 6
#define TK_FLOAT 7
#define TK_DOUBLE 8
#define TK_STRING 9
#define TK_TIMESTAMP 10
#define TK_BINARY 11
#define TK_NCHAR 12
#define TK_OR 13
#define TK_AND 14
#define TK_NOT 15
#define TK_EQ 16
#define TK_NE 17
#define TK_ISNULL 18
#define TK_NOTNULL 19
#define TK_IS 20
#define TK_LIKE 21
#define TK_MATCH 22
#define TK_NMATCH 23
#define TK_GLOB 24
#define TK_BETWEEN 25
#define TK_IN 26
#define TK_GT 27
#define TK_GE 28
#define TK_LT 29
#define TK_LE 30
#define TK_BITAND 31
#define TK_BITOR 32
#define TK_LSHIFT 33
#define TK_RSHIFT 34
#define TK_PLUS 35
#define TK_MINUS 36
#define TK_DIVIDE 37
#define TK_TIMES 38
#define TK_STAR 39
#define TK_SLASH 40
#define TK_REM 41
#define TK_CONCAT 42
#define TK_UMINUS 43
#define TK_UPLUS 44
#define TK_BITNOT 45
#define TK_SHOW 46
#define TK_DATABASES 47
#define TK_TOPICS 48
#define TK_FUNCTIONS 49
#define TK_MNODES 50
#define TK_DNODES 51
#define TK_ACCOUNTS 52
#define TK_USERS 53
#define TK_MODULES 54
#define TK_QUERIES 55
#define TK_CONNECTIONS 56
#define TK_STREAMS 57
#define TK_VARIABLES 58
#define TK_SCORES 59
#define TK_GRANTS 60
#define TK_VNODES 61
#define TK_DOT 62
#define TK_CREATE 63
#define TK_TABLE 64
#define TK_STABLE 65
#define TK_DATABASE 66
#define TK_TABLES 67
#define TK_STABLES 68
#define TK_VGROUPS 69
#define TK_DROP 70
#define TK_TOPIC 71
#define TK_FUNCTION 72
#define TK_DNODE 73
#define TK_USER 74
#define TK_ACCOUNT 75
#define TK_USE 76
#define TK_DESCRIBE 77
#define TK_DESC 78
#define TK_ALTER 79
#define TK_PASS 80
#define TK_PRIVILEGE 81
#define TK_LOCAL 82
#define TK_COMPACT 83
#define TK_LP 84
#define TK_RP 85
#define TK_IF 86
#define TK_EXISTS 87
#define TK_AS 88
#define TK_OUTPUTTYPE 89
#define TK_AGGREGATE 90
#define TK_BUFSIZE 91
#define TK_PPS 92
#define TK_TSERIES 93
#define TK_DBS 94
#define TK_STORAGE 95
#define TK_QTIME 96
#define TK_CONNS 97
#define TK_STATE 98
#define TK_COMMA 99
#define TK_KEEP 100
#define TK_CACHE 101
#define TK_REPLICA 102
#define TK_QUORUM 103
#define TK_DAYS 104
#define TK_MINROWS 105
#define TK_MAXROWS 106
#define TK_BLOCKS 107
#define TK_CTIME 108
#define TK_WAL 109
#define TK_FSYNC 110
#define TK_COMP 111
#define TK_PRECISION 112
#define TK_UPDATE 113
#define TK_CACHELAST 114
#define TK_PARTITIONS 115
#define TK_UNSIGNED 116
#define TK_TAGS 117
#define TK_USING 118
#define TK_NULL 119
#define TK_NOW 120
#define TK_SELECT 121
#define TK_UNION 122
#define TK_ALL 123
#define TK_DISTINCT 124
#define TK_FROM 125
#define TK_VARIABLE 126
#define TK_INTERVAL 127
#define TK_EVERY 128
#define TK_SESSION 129
#define TK_STATE_WINDOW 130
#define TK_FILL 131
#define TK_SLIDING 132
#define TK_ORDER 133
#define TK_BY 134
#define TK_ASC 135
#define TK_GROUP 136
#define TK_HAVING 137
#define TK_LIMIT 138
#define TK_OFFSET 139
#define TK_SLIMIT 140
#define TK_SOFFSET 141
#define TK_WHERE 142
#define TK_RESET 143
#define TK_QUERY 144
#define TK_SYNCDB 145
#define TK_ADD 146
#define TK_COLUMN 147
#define TK_MODIFY 148
#define TK_TAG 149
#define TK_CHANGE 150
#define TK_SET 151
#define TK_KILL 152
#define TK_CONNECTION 153
#define TK_STREAM 154
#define TK_COLON 155
#define TK_ABORT 156
#define TK_AFTER 157
#define TK_ATTACH 158
#define TK_BEFORE 159
#define TK_BEGIN 160
#define TK_CASCADE 161
#define TK_CLUSTER 162
#define TK_CONFLICT 163
#define TK_COPY 164
#define TK_DEFERRED 165
#define TK_DELIMITERS 166
#define TK_DETACH 167
#define TK_EACH 168
#define TK_END 169
#define TK_EXPLAIN 170
#define TK_FAIL 171
#define TK_FOR 172
#define TK_IGNORE 173
#define TK_IMMEDIATE 174
#define TK_INITIALLY 175
#define TK_INSTEAD 176
#define TK_KEY 177
#define TK_OF 178
#define TK_RAISE 179
#define TK_REPLACE 180
#define TK_RESTRICT 181
#define TK_ROW 182
#define TK_STATEMENT 183
#define TK_TRIGGER 184
#define TK_VIEW 185
#define TK_IPTOKEN 186
#define TK_SEMI 187
#define TK_NONE 188
#define TK_PREV 189
#define TK_LINEAR 190
#define TK_IMPORT 191
#define TK_TBNAME 192
#define TK_JOIN 193
#define TK_INSERT 194
#define TK_INTO 195
#define TK_VALUES 196
#define TK_INTEGER 3
#define TK_FLOAT 4
#define TK_STRING 5
#define TK_TIMESTAMP 6
#define TK_OR 7
#define TK_AND 8
#define TK_NOT 9
#define TK_EQ 10
#define TK_NE 11
#define TK_ISNULL 12
#define TK_NOTNULL 13
#define TK_IS 14
#define TK_LIKE 15
#define TK_MATCH 16
#define TK_NMATCH 17
#define TK_GLOB 18
#define TK_BETWEEN 19
#define TK_IN 20
#define TK_GT 21
#define TK_GE 22
#define TK_LT 23
#define TK_LE 24
#define TK_BITAND 25
#define TK_BITOR 26
#define TK_LSHIFT 27
#define TK_RSHIFT 28
#define TK_PLUS 29
#define TK_MINUS 30
#define TK_DIVIDE 31
#define TK_TIMES 32
#define TK_STAR 33
#define TK_SLASH 34
#define TK_REM 35
#define TK_CONCAT 36
#define TK_UMINUS 37
#define TK_UPLUS 38
#define TK_BITNOT 39
#define TK_SHOW 40
#define TK_DATABASES 41
#define TK_TOPICS 42
#define TK_FUNCTIONS 43
#define TK_MNODES 44
#define TK_DNODES 45
#define TK_ACCOUNTS 46
#define TK_USERS 47
#define TK_MODULES 48
#define TK_QUERIES 49
#define TK_CONNECTIONS 50
#define TK_STREAMS 51
#define TK_VARIABLES 52
#define TK_SCORES 53
#define TK_GRANTS 54
#define TK_VNODES 55
#define TK_DOT 56
#define TK_CREATE 57
#define TK_TABLE 58
#define TK_STABLE 59
#define TK_DATABASE 60
#define TK_TABLES 61
#define TK_STABLES 62
#define TK_VGROUPS 63
#define TK_DROP 64
#define TK_TOPIC 65
#define TK_FUNCTION 66
#define TK_DNODE 67
#define TK_USER 68
#define TK_ACCOUNT 69
#define TK_USE 70
#define TK_DESCRIBE 71
#define TK_DESC 72
#define TK_ALTER 73
#define TK_PASS 74
#define TK_PRIVILEGE 75
#define TK_LOCAL 76
#define TK_COMPACT 77
#define TK_LP 78
#define TK_RP 79
#define TK_IF 80
#define TK_EXISTS 81
#define TK_PORT 82
#define TK_IPTOKEN 83
#define TK_AS 84
#define TK_OUTPUTTYPE 85
#define TK_AGGREGATE 86
#define TK_BUFSIZE 87
#define TK_PPS 88
#define TK_TSERIES 89
#define TK_DBS 90
#define TK_STORAGE 91
#define TK_QTIME 92
#define TK_CONNS 93
#define TK_STATE 94
#define TK_COMMA 95
#define TK_KEEP 96
#define TK_CACHE 97
#define TK_REPLICA 98
#define TK_QUORUM 99
#define TK_DAYS 100
#define TK_MINROWS 101
#define TK_MAXROWS 102
#define TK_BLOCKS 103
#define TK_CTIME 104
#define TK_WAL 105
#define TK_FSYNC 106
#define TK_COMP 107
#define TK_PRECISION 108
#define TK_UPDATE 109
#define TK_CACHELAST 110
#define TK_UNSIGNED 111
#define TK_TAGS 112
#define TK_USING 113
#define TK_NULL 114
#define TK_NOW 115
#define TK_SELECT 116
#define TK_UNION 117
#define TK_ALL 118
#define TK_DISTINCT 119
#define TK_FROM 120
#define TK_VARIABLE 121
#define TK_INTERVAL 122
#define TK_EVERY 123
#define TK_SESSION 124
#define TK_STATE_WINDOW 125
#define TK_FILL 126
#define TK_SLIDING 127
#define TK_ORDER 128
#define TK_BY 129
#define TK_ASC 130
#define TK_GROUP 131
#define TK_HAVING 132
#define TK_LIMIT 133
#define TK_OFFSET 134
#define TK_SLIMIT 135
#define TK_SOFFSET 136
#define TK_WHERE 137
#define TK_RESET 138
#define TK_QUERY 139
#define TK_SYNCDB 140
#define TK_ADD 141
#define TK_COLUMN 142
#define TK_MODIFY 143
#define TK_TAG 144
#define TK_CHANGE 145
#define TK_SET 146
#define TK_KILL 147
#define TK_CONNECTION 148
#define TK_STREAM 149
#define TK_COLON 150
#define TK_ABORT 151
#define TK_AFTER 152
#define TK_ATTACH 153
#define TK_BEFORE 154
#define TK_BEGIN 155
#define TK_CASCADE 156
#define TK_CLUSTER 157
#define TK_CONFLICT 158
#define TK_COPY 159
#define TK_DEFERRED 160
#define TK_DELIMITERS 161
#define TK_DETACH 162
#define TK_EACH 163
#define TK_END 164
#define TK_EXPLAIN 165
#define TK_FAIL 166
#define TK_FOR 167
#define TK_IGNORE 168
#define TK_IMMEDIATE 169
#define TK_INITIALLY 170
#define TK_INSTEAD 171
#define TK_KEY 172
#define TK_OF 173
#define TK_RAISE 174
#define TK_REPLACE 175
#define TK_RESTRICT 176
#define TK_ROW 177
#define TK_STATEMENT 178
#define TK_TRIGGER 179
#define TK_VIEW 180
#define TK_SEMI 181
#define TK_NONE 182
#define TK_PREV 183
#define TK_LINEAR 184
#define TK_IMPORT 185
#define TK_TBNAME 186
#define TK_JOIN 187
#define TK_INSERT 188
#define TK_INTO 189
#define TK_VALUES 190
#define TK_SPACE 300
@ -223,6 +217,6 @@
#define TK_FILE 306
#define TK_QUESTION 307 // denoting the placeholder of "?",when invoking statement bind query
#endif /*_TD_COMMON_TOKEN_DEF_H_*/
#endif

10
include/dnode/bnode/bnode.h
View File

@ -49,10 +49,11 @@ typedef struct {
/**
* @brief Start one Bnode in Dnode.
*
* @param path Path of the bnode.
* @param pOption Option of the bnode.
* @return SBnode* The bnode object.
*/
SBnode *bndOpen(const SBnodeOpt *pOption);
SBnode *bndOpen(const char *path, const SBnodeOpt *pOption);
/**
* @brief Stop Bnode in Dnode.
@ -79,6 +80,13 @@ int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad);
*/
int32_t bndProcessWMsgs(SBnode *pBnode, SArray *pMsgs);
/**
* @brief Drop a bnode.
*
* @param path Path of the bnode.
*/
void bndDestroy(const char *path);
#ifdef __cplusplus
}
#endif

20
include/dnode/mnode/mnode.h
View File

@ -147,28 +147,12 @@ void mndCleanupMsg(SMnodeMsg *pMsg);
void mndSendRsp(SMnodeMsg *pMsg, int32_t code);
/**
* @brief Process the read request.
* @brief Process the read, write, sync request.
*
* @param pMsg The request msg.
* @return int32_t 0 for success, -1 for failure.
*/
void mndProcessReadMsg(SMnodeMsg *pMsg);
/**
* @brief Process the write request.
*
* @param pMsg The request msg.
* @return int32_t 0 for success, -1 for failure.
*/
void mndProcessWriteMsg(SMnodeMsg *pMsg);
/**
* @brief Process the sync request.
*
* @param pMsg The request msg.
* @return int32_t 0 for success, -1 for failure.
*/
void mndProcessSyncMsg(SMnodeMsg *pMsg);
void mndProcessMsg(SMnodeMsg *pMsg);
#ifdef __cplusplus
}

14
include/dnode/qnode/qnode.h
View File

@ -78,24 +78,14 @@ void qndClose(SQnode *pQnode);
int32_t qndGetLoad(SQnode *pQnode, SQnodeLoad *pLoad);
/**
* @brief Process a query message.
* @brief Process a query or fetch message.
*
* @param pQnode The qnode object.
* @param pMsg The request message
* @param pRsp The response message
* @return int32_t 0 for success, -1 for failure
*/
int32_t qndProcessQueryReq(SQnode *pQnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
/**
* @brief Process a fetch message.
*
* @param pQnode The qnode object.
* @param pMsg The request message
* @param pRsp The response message
* @return int32_t 0 for success, -1 for failure
*/
int32_t qndProcessFetchReq(SQnode *pQnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
int32_t qndProcessMsg(SQnode *pQnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
#ifdef __cplusplus
}

12
include/dnode/snode/snode.h
View File

@ -49,10 +49,11 @@ typedef struct {
/**
* @brief Start one Snode in Dnode.
*
* @param path Path of the snode.
* @param pOption Option of the snode.
* @return SSnode* The snode object.
*/
SSnode *sndOpen(const SSnodeOpt *pOption);
SSnode *sndOpen(const char *path, const SSnodeOpt *pOption);
/**
* @brief Stop Snode in Dnode.
@ -78,7 +79,14 @@ int32_t sndGetLoad(SSnode *pSnode, SSnodeLoad *pLoad);
* @param pRsp The response message
* @return int32_t 0 for success, -1 for failure
*/
int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
int32_t sndProcessMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp);
/**
* @brief Drop a snode.
*
* @param path Path of the snode.
*/
void sndDestroy(const char *path);
#ifdef __cplusplus
}

2
include/dnode/vnode/tq/tq.h
View File

@ -251,6 +251,8 @@ typedef struct STqMetaStore {
STqMetaList* bucket[TQ_BUCKET_SIZE];
// a table head
STqMetaList* unpersistHead;
// topics that are not connectted
STqMetaList* unconnectTopic;
// TODO:temporaral use, to be replaced by unified tfile
int fileFd;

4
include/libs/catalog/catalog.h
View File

@ -67,14 +67,14 @@ int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName
/**
* Get a table's meta data.
* @param pCatalog (input, got with catalogGetHandle)
* @param pRpc (input, rpc object)
* @param pTransporter (input, rpc object)
* @param pMgmtEps (input, mnode EPs)
* @param pDBName (input, full db name)
* @param pTableName (input, table name, NOT including db name)
* @param pTableMeta(output, table meta data, NEED to free it by calller)
* @return error code
*/
int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta);
int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void * pTransporter, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta);
/**
* Force renew a table's local cached meta data.

10
include/libs/parser/parsenodes.h
View File

@ -44,9 +44,12 @@ typedef struct SField {
} SField;
typedef struct SParseBasicCtx {
const char *db;
int32_t acctId;
uint64_t requestId;
uint64_t requestId;
int32_t acctId;
const char *db;
void *pTransporter;
SEpSet mgmtEpSet;
struct SCatalog *pCatalog;
} SParseBasicCtx;
typedef struct SFieldInfo {
@ -163,6 +166,7 @@ typedef struct SInsertStmtInfo {
typedef struct SDclStmtInfo {
int16_t nodeType;
int16_t msgType;
SEpSet epSet;
char* pMsg;
int32_t msgLen;
} SDclStmtInfo;

9
include/libs/parser/parser.h
View File

@ -23,10 +23,7 @@ extern "C" {
#include "parsenodes.h"
typedef struct SParseContext {
SParseBasicCtx ctx;
void *pRpc;
struct SCatalog *pCatalog;
const SEpSet *pEpSet;
SParseBasicCtx ctx;
int8_t schemaAttached; // denote if submit block is built with table schema or not
const char *pSql; // sql string
size_t sqlLen; // length of the sql string
@ -46,7 +43,7 @@ int32_t qParseQuerySql(SParseContext* pContext, SQueryNode** pQuery);
bool qIsDdlQuery(const SQueryNode* pQuery);
void qDestoryQuery(SQueryNode* pQuery);
void qDestroyQuery(SQueryNode* pQuery);
/**
* Convert a normal sql statement to only query tags information to enable that the subscribe client can be aware quickly of the true vgroup ids that
@ -86,4 +83,4 @@ void addIntoSourceParam(SSourceParam* pSourceParam, tExprNode* pNode, SColumn* p
}
#endif
#endif /*_TD_PARSER_H_*/
#endif /*_TD_PARSER_H_*/

8
include/libs/planner/planner.h
View File

@ -21,6 +21,7 @@ extern "C" {
#endif
#include "tmsg.h"
#include "tarray.h"
#define QUERY_TYPE_MERGE 1
#define QUERY_TYPE_PARTIAL 2
@ -131,7 +132,7 @@ typedef struct SSubplan {
typedef struct SQueryDag {
uint64_t queryId;
int32_t numOfSubplans;
SArray *pSubplans; // Element is SArray*, and nested element is SSubplan. The execution level of subplan, starting from 0.
SArray *pSubplans; // SArray*<SArray*<SSubplan*>>. The execution level of subplan, starting from 0.
} SQueryDag;
struct SQueryNode;
@ -165,8 +166,11 @@ void qDestroySubplan(SSubplan* pSubplan);
*/
void qDestroyQueryDag(SQueryDag* pDag);
char* qDagToString(const SQueryDag* pDag);
SQueryDag* qStringToDag(const char* pStr);
#ifdef __cplusplus
}
#endif
#endif /*_TD_PLANNER_H_*/
#endif /*_TD_PLANNER_H_*/

2
include/libs/qcom/query.h
View File

@ -81,7 +81,7 @@ typedef struct SDBVgroupInfo {
} SDBVgroupInfo;
typedef struct SUseDbOutput {
char db[TSDB_FULL_DB_NAME_LEN];
char db[TSDB_DB_FNAME_LEN];
SDBVgroupInfo dbVgroup;
} SUseDbOutput;

1
include/util/taoserror.h
View File

@ -69,6 +69,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_CHECKSUM_ERROR TAOS_DEF_ERROR_CODE(0, 0x0107)
#define TSDB_CODE_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x0108)
#define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0109)
#define TSDB_CODE_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x010A)
#define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0110)
#define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0111)
#define TSDB_CODE_REF_ID_REMOVED TAOS_DEF_ERROR_CODE(0, 0x0112)

24
include/util/tdef.h
View File

@ -149,7 +149,7 @@ do { \
#define IS_RELATION_OPTR(op) (((op) >= TSDB_RELATION_LESS) && ((op) < TSDB_RELATION_IN))
#define IS_ARITHMETIC_OPTR(op) (((op) >= TSDB_BINARY_OP_ADD) && ((op) <= TSDB_BINARY_OP_REMAINDER))
#define TS_PATH_DELIMITER_LEN 1
#define TSDB_NAME_DELIMITER_LEN 1
#define TSDB_UNI_LEN 24
#define TSDB_USER_LEN TSDB_UNI_LEN
@ -165,17 +165,17 @@ do { \
#define TSDB_TABLE_NAME_LEN 193 // it is a null-terminated string
#define TSDB_TOPIC_NAME_LEN 193 // it is a null-terminated string
#define TSDB_DB_NAME_LEN 65
#define TSDB_FULL_DB_NAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN)
#define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_FUNC_NAME_LEN 65
#define TSDB_FUNC_COMMENT_LEN 4096
#define TSDB_FUNC_CODE_LEN (65535 - 512)
#define TSDB_FUNC_BUF_SIZE 512
#define TSDB_FUNC_TYPE_SCALAR 1
#define TSDB_FUNC_TYPE_AGGREGATE 2
#define TSDB_FUNC_NAME_LEN 65
#define TSDB_FUNC_COMMENT_LEN 4096
#define TSDB_FUNC_CODE_LEN (65535 - 512)
#define TSDB_FUNC_BUF_SIZE 512
#define TSDB_FUNC_TYPE_SCALAR 1
#define TSDB_FUNC_TYPE_AGGREGATE 2
#define TSDB_TYPE_STR_MAX_LEN 32
#define TSDB_TABLE_FNAME_LEN (TSDB_FULL_DB_NAME_LEN + TSDB_TABLE_NAME_LEN)
#define TSDB_TABLE_FNAME_LEN (TSDB_DB_FNAME_LEN + TSDB_TABLE_NAME_LEN + TSDB_NAME_DELIMITER_LEN)
#define TSDB_TOPIC_FNAME_LEN TSDB_TABLE_FNAME_LEN
#define TSDB_COL_NAME_LEN 65
#define TSDB_MAX_SAVED_SQL_LEN TSDB_MAX_COLUMNS * 64
@ -317,12 +317,12 @@ do { \
#define TSDB_MAX_FIELD_LEN 16384
#define TSDB_MAX_BINARY_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384
#define TSDB_MAX_NCHAR_LEN (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384
#define PRIMARYKEY_TIMESTAMP_COL_ID 0
#define PRIMARYKEY_TIMESTAMP_COL_ID 1
#define TSDB_MAX_RPC_THREADS 5
#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type
#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode
#define TSDB_QUERY_TYPE_NON_TYPE 0x00u // none type
#define TSDB_QUERY_TYPE_FREE_RESOURCE 0x01u // free qhandle at vnode
#define TSDB_META_COMPACT_RATIO 0 // disable tsdb meta compact by default

20
source/client/inc/clientInt.h
View File

@ -75,16 +75,16 @@ typedef struct SAppInfo {
} SAppInfo;
typedef struct STscObj {
char user[TSDB_USER_LEN];
char pass[TSDB_PASSWORD_LEN];
char db[TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN];
int32_t acctId;
uint32_t connId;
uint64_t id; // ref ID returned by taosAddRef
void *pTransporter;
pthread_mutex_t mutex; // used to protect the operation on db
int32_t numOfReqs; // number of sqlObj from this tscObj
SAppInstInfo *pAppInfo;
char user[TSDB_USER_LEN];
char pass[TSDB_PASSWORD_LEN];
char db[TSDB_DB_FNAME_LEN];
int32_t acctId;
uint32_t connId;
uint64_t id; // ref ID returned by taosAddRef
void *pTransporter;
pthread_mutex_t mutex; // used to protect the operation on db
int32_t numOfReqs; // number of sqlObj from this tscObj
SAppInstInfo *pAppInfo;
} STscObj;
typedef struct SReqResultInfo {

2
source/client/src/clientEnv.c
View File

@ -227,7 +227,7 @@ void taos_init_imp(void) {
rpcInit();
SCatalogCfg cfg = {.enableVgroupCache = true, .maxDBCacheNum = 100, .maxTblCacheNum = 100};
SCatalogCfg cfg = {.maxDBCacheNum = 100, .maxTblCacheNum = 100};
catalogInit(&cfg);
tscDebug("starting to initialize TAOS driver, local ep: %s", tsLocalEp);

144
source/client/src/clientImpl.c
View File

@ -145,15 +145,32 @@ int32_t buildRequest(STscObj *pTscObj, const char *sql, int sqlLen, SRequestObj*
}
int32_t parseSql(SRequestObj* pRequest, SQueryNode** pQuery) {
STscObj* pTscObj = pRequest->pTscObj;
SParseContext cxt = {
.ctx = {.requestId = pRequest->requestId, .acctId = pRequest->pTscObj->acctId, .db = getConnectionDB(pRequest->pTscObj)},
.pSql = pRequest->sqlstr,
.ctx = {.requestId = pRequest->requestId, .acctId = pTscObj->acctId, .db = getConnectionDB(pTscObj), .pTransporter = pTscObj->pTransporter},
.pSql = pRequest->sqlstr,
.sqlLen = pRequest->sqlLen,
.pMsg = pRequest->msgBuf,
.pMsg = pRequest->msgBuf,
.msgLen = ERROR_MSG_BUF_DEFAULT_SIZE
};
int32_t code = qParseQuerySql(&cxt, pQuery);
cxt.ctx.mgmtEpSet = getEpSet_s(&pTscObj->pAppInfo->mgmtEp);
// todo OPT performance
char buf[12] = {0};
sprintf(buf, "%"PRId64, pTscObj->pAppInfo->clusterId);
struct SCatalog* pCatalog = NULL;
int32_t code = catalogGetHandle(buf, &pCatalog);
if (code != TSDB_CODE_SUCCESS) {
tfree(cxt.ctx.db);
return code;
}
cxt.ctx.pCatalog = pCatalog;
code = qParseQuerySql(&cxt, pQuery);
tfree(cxt.ctx.db);
return code;
}
@ -165,47 +182,47 @@ int32_t execDdlQuery(SRequestObj* pRequest, SQueryNode* pQuery) {
STscObj* pTscObj = pRequest->pTscObj;
SMsgSendInfo* body = buildSendMsgInfoImpl(pRequest);
SMsgSendInfo* pSendMsg = buildSendMsgInfoImpl(pRequest);
SEpSet* pEpSet = &pTscObj->pAppInfo->mgmtEp.epSet;
if (pDcl->msgType == TDMT_VND_CREATE_TABLE) {
struct SCatalog* pCatalog = NULL;
char buf[18] = {0};
sprintf(buf, "%" PRId64, pRequest->pTscObj->pAppInfo->clusterId);
int32_t code = catalogGetHandle(buf, &pCatalog);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
SCreateTableMsg* pMsg = body->msgInfo.pData;
SName t = {0};
tNameFromString(&t, pMsg->name, T_NAME_ACCT|T_NAME_DB|T_NAME_TABLE);
char db[TSDB_DB_NAME_LEN + TS_PATH_DELIMITER_LEN + TSDB_ACCT_ID_LEN] = {0};
tNameGetFullDbName(&t, db);
SVgroupInfo info = {0};
catalogGetTableHashVgroup(pCatalog, pRequest->pTscObj->pTransporter, pEpSet, db, tNameGetTableName(&t), &info);
// struct SCatalog* pCatalog = NULL;
//
// char buf[18] = {0};
// sprintf(buf, "%" PRId64, pRequest->pTscObj->pAppInfo->clusterId);
// int32_t code = catalogGetHandle(buf, &pCatalog);
// if (code != TSDB_CODE_SUCCESS) {
// return code;
// }
//
// SCreateTableMsg* pMsg = pSendMsg->msgInfo.pData;
//
// SName t = {0};
// tNameFromString(&t, pMsg->name, T_NAME_ACCT|T_NAME_DB|T_NAME_TABLE);
//
// char db[TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN + TSDB_ACCT_ID_LEN] = {0};
// tNameGetFullDbName(&t, db);
//
// SVgroupInfo info = {0};
// catalogGetTableHashVgroup(pCatalog, pRequest->pTscObj->pTransporter, pEpSet, db, tNameGetTableName(&t), &info);
//
int64_t transporterId = 0;
SEpSet ep = {0};
ep.inUse = info.inUse;
ep.numOfEps = info.numOfEps;
for(int32_t i = 0; i < ep.numOfEps; ++i) {
ep.port[i] = info.epAddr[i].port;
tstrncpy(ep.fqdn[i], info.epAddr[i].fqdn, tListLen(ep.fqdn[i]));
}
asyncSendMsgToServer(pTscObj->pTransporter, &ep, &transporterId, body);
// SEpSet ep = {0};
// ep.inUse = info.inUse;
// ep.numOfEps = info.numOfEps;
// for(int32_t i = 0; i < ep.numOfEps; ++i) {
// ep.port[i] = info.epAddr[i].port;
// tstrncpy(ep.fqdn[i], info.epAddr[i].fqdn, tListLen(ep.fqdn[i]));
// }
asyncSendMsgToServer(pTscObj->pTransporter, &pDcl->epSet, &transporterId, pSendMsg);
} else {
int64_t transporterId = 0;
asyncSendMsgToServer(pTscObj->pTransporter, pEpSet, &transporterId, body);
asyncSendMsgToServer(pTscObj->pTransporter, pEpSet, &transporterId, pSendMsg);
}
tsem_wait(&pRequest->body.rspSem);
destroySendMsgInfo(body);
destroySendMsgInfo(pSendMsg);
return TSDB_CODE_SUCCESS;
}
@ -213,6 +230,57 @@ int32_t scheduleQuery(SRequestObj* pRequest, SQueryDag* pDag, void** pJob) {
return scheduleAsyncExecJob(pRequest->pTscObj->pTransporter, NULL/*todo appInfo.xxx*/, pDag, pJob);
}
TAOS_RES *tmq_create_topic(TAOS* taos, const char* name, const char* sql, int sqlLen) {
STscObj* pTscObj = (STscObj*)taos;
SRequestObj* pRequest = NULL;
SQueryNode* pQuery = NULL;
SQueryDag* pDag = NULL;
char *dagStr = NULL;
terrno = TSDB_CODE_SUCCESS;
CHECK_CODE_GOTO(buildRequest(pTscObj, sql, sqlLen, &pRequest), _return);
CHECK_CODE_GOTO(parseSql(pRequest, &pQuery), _return);
//TODO: check sql valid
CHECK_CODE_GOTO(qCreateQueryDag(pQuery, &pDag), _return);
dagStr = qDagToString(pDag);
if(dagStr == NULL) {
//TODO
}
SCMCreateTopicReq req = {
.name = (char*)name,
.igExists = 0,
.phyPlan = dagStr,
};
void* buf = NULL;
int tlen = tSerializeSCMCreateTopicReq(&buf, &req);
pRequest->body.requestMsg = (SDataBuf){ .pData = buf, .len = tlen };
SMsgSendInfo* body = buildSendMsgInfoImpl(pRequest);
SEpSet* pEpSet = &pTscObj->pAppInfo->mgmtEp.epSet;
int64_t transporterId = 0;
asyncSendMsgToServer(pTscObj->pTransporter, pEpSet, &transporterId, body);
tsem_wait(&pRequest->body.rspSem);
destroySendMsgInfo(body);
_return:
qDestroyQuery(pQuery);
qDestroyQueryDag(pDag);
destroySendMsgInfo(body);
if (pRequest != NULL && terrno != TSDB_CODE_SUCCESS) {
pRequest->code = terrno;
}
return pRequest;
}
TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) {
STscObj *pTscObj = (STscObj *)taos;
if (sqlLen > (size_t) tsMaxSQLStringLen) {
@ -240,7 +308,7 @@ TAOS_RES *taos_query_l(TAOS *taos, const char *sql, int sqlLen) {
}
_return:
qDestoryQuery(pQuery);
qDestroyQuery(pQuery);
qDestroyQueryDag(pDag);
if (NULL != pRequest && TSDB_CODE_SUCCESS != terrno) {
pRequest->code = terrno;
@ -332,7 +400,7 @@ static SMsgSendInfo* buildConnectMsg(SRequestObj *pRequest) {
pMsgSendInfo->msgInfo.len = sizeof(SConnectMsg);
pMsgSendInfo->requestObjRefId = pRequest->self;
pMsgSendInfo->requestId = pRequest->requestId;
pMsgSendInfo->fp = handleRequestRspFp[pMsgSendInfo->msgType];
pMsgSendInfo->fp = handleRequestRspFp[TMSG_INDEX(pMsgSendInfo->msgType)];
pMsgSendInfo->param = pRequest;
SConnectMsg *pConnect = calloc(1, sizeof(SConnectMsg));

20
source/client/src/clientMsgHandler.c
View File

@ -79,7 +79,7 @@ static int32_t buildRetrieveMnodeMsg(SRequestObj *pRequest, SMsgSendInfo* pMsgSe
pMsgSendInfo->msgInfo.len = sizeof(SRetrieveTableMsg);
pMsgSendInfo->requestObjRefId = pRequest->self;
pMsgSendInfo->param = pRequest;
pMsgSendInfo->fp = handleRequestRspFp[pMsgSendInfo->msgType];
pMsgSendInfo->fp = handleRequestRspFp[TMSG_INDEX(pMsgSendInfo->msgType)];
SRetrieveTableMsg *pRetrieveMsg = calloc(1, sizeof(SRetrieveTableMsg));
if (pRetrieveMsg == NULL) {
@ -104,7 +104,7 @@ SMsgSendInfo* buildSendMsgInfoImpl(SRequestObj *pRequest) {
pMsgSendInfo->requestId = pRequest->requestId;
pMsgSendInfo->param = pRequest;
pMsgSendInfo->fp = (handleRequestRspFp[pRequest->type] == NULL)? genericRspCallback:handleRequestRspFp[pRequest->type];
pMsgSendInfo->fp = (handleRequestRspFp[TMSG_INDEX(pRequest->type)] == NULL)? genericRspCallback:handleRequestRspFp[TMSG_INDEX(pRequest->type)];
}
return pMsgSendInfo;
@ -139,7 +139,7 @@ int32_t processShowRsp(void* param, const SDataBuf* pMsg, int32_t code) {
pFields[i].bytes = pSchema[i].bytes;
}
// pRequest->body.resInfo.pRspMsg = pMsg->pData;
pRequest->body.resInfo.pRspMsg = pMsg->pData;
SReqResultInfo* pResInfo = &pRequest->body.resInfo;
pResInfo->fields = pFields;
@ -290,11 +290,11 @@ void initMsgHandleFp() {
tscProcessMsgRsp[TSDB_SQL_SHOW_CREATE_DATABASE] = tscProcessShowCreateRsp;
#endif
handleRequestRspFp[TDMT_MND_CONNECT] = processConnectRsp;
handleRequestRspFp[TDMT_MND_SHOW] = processShowRsp;
handleRequestRspFp[TDMT_MND_SHOW_RETRIEVE] = processRetrieveMnodeRsp;
handleRequestRspFp[TDMT_MND_CREATE_DB] = processCreateDbRsp;
handleRequestRspFp[TDMT_MND_USE_DB] = processUseDbRsp;
handleRequestRspFp[TDMT_MND_CREATE_STB] = processCreateTableRsp;
handleRequestRspFp[TDMT_MND_DROP_DB] = processDropDbRsp;
handleRequestRspFp[TMSG_INDEX(TDMT_MND_CONNECT)] = processConnectRsp;
handleRequestRspFp[TMSG_INDEX(TDMT_MND_SHOW)] = processShowRsp;
handleRequestRspFp[TMSG_INDEX(TDMT_MND_SHOW_RETRIEVE)] = processRetrieveMnodeRsp;
handleRequestRspFp[TMSG_INDEX(TDMT_MND_CREATE_DB)] = processCreateDbRsp;
handleRequestRspFp[TMSG_INDEX(TDMT_MND_USE_DB)] = processUseDbRsp;
handleRequestRspFp[TMSG_INDEX(TDMT_MND_CREATE_STB)] = processCreateTableRsp;
handleRequestRspFp[TMSG_INDEX(TDMT_MND_DROP_DB)] = processDropDbRsp;
}

49
source/client/test/clientTests.cpp
View File

@ -177,14 +177,14 @@ TEST(testCase, create_dnode_Test) {
if (taos_errno(pRes) != 0) {
printf("error in create dnode, reason:%s\n", taos_errstr(pRes));
}
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
ASSERT_TRUE(pFields == NULL);
int32_t numOfFields = taos_num_fields(pRes);
ASSERT_EQ(numOfFields, 0);
taos_free_result(pRes);
pRes = taos_query(pConn, "create dnode 1.1.1.1 port 9000");
if (taos_errno(pRes) != 0) {
printf("failed to create dnode, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
taos_close(pConn);
}
@ -399,6 +399,41 @@ TEST(testCase, drop_stable_Test) {
taos_close(pConn);
}
TEST(testCase, create_topic_Test) {
TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
assert(pConn != NULL);
TAOS_RES* pRes = taos_query(pConn, "create database abc1");
if (taos_errno(pRes) != 0) {
printf("error in create db, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
pRes = taos_query(pConn, "use abc1");
if (taos_errno(pRes) != 0) {
printf("error in use db, reason:%s\n", taos_errstr(pRes));
}
taos_free_result(pRes);
pRes = taos_query(pConn, "create stable st1(ts timestamp, k int) tags(a int)");
if (taos_errno(pRes) != 0) {
printf("error in create stable, reason:%s\n", taos_errstr(pRes));
}
TAOS_FIELD* pFields = taos_fetch_fields(pRes);
ASSERT_TRUE(pFields == NULL);
int32_t numOfFields = taos_num_fields(pRes);
ASSERT_EQ(numOfFields, 0);
taos_free_result(pRes);
char* sql = "select * from st1";
tmq_create_topic(pConn, "test_topic_1", sql, strlen(sql));
taos_close(pConn);
}
//TEST(testCase, show_table_Test) {
// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
// assert(pConn != NULL);

9
source/common/src/tep.c
View File

@ -39,3 +39,12 @@ void updateEpSet_s(SCorEpSet *pEpSet, SEpSet *pNewEpSet) {
taosCorEndWrite(&pEpSet->version);
}
SEpSet getEpSet_s(SCorEpSet *pEpSet) {
SEpSet ep = {0};
taosCorBeginRead(&pEpSet->version);
ep = pEpSet->epSet;
taosCorEndRead(&pEpSet->version);
return ep;
}

9
source/common/src/tname.c
View File

@ -110,7 +110,7 @@ int32_t tNameExtractFullName(const SName* name, char* dst) {
return -1;
}
int32_t len = snprintf(dst, TSDB_FULL_DB_NAME_LEN, "%d.%s", name->acctId, name->dbname);
int32_t len = snprintf(dst, TSDB_DB_FNAME_LEN, "%d.%s", name->acctId, name->dbname);
size_t tnameLen = strlen(name->tname);
if (tnameLen > 0) {
@ -134,10 +134,10 @@ int32_t tNameLen(const SName* name) {
if (name->type == TSDB_DB_NAME_T) {
assert(len2 == 0);
return len + len1 + TS_PATH_DELIMITER_LEN;
return len + len1 + TSDB_NAME_DELIMITER_LEN;
} else {
assert(len2 > 0);
return len + len1 + len2 + TS_PATH_DELIMITER_LEN * 2;
return len + len1 + len2 + TSDB_NAME_DELIMITER_LEN * 2;
}
}
@ -171,8 +171,7 @@ int32_t tNameGetDbName(const SName* name, char* dst) {
int32_t tNameGetFullDbName(const SName* name, char* dst) {
assert(name != NULL && dst != NULL);
snprintf(dst, TSDB_ACCT_ID_LEN + TS_PATH_DELIMITER_LEN + TSDB_DB_NAME_LEN, // there is a over write risk
"%d.%s", name->acctId, name->dbname);
snprintf(dst, TSDB_DB_FNAME_LEN, "%d.%s", name->acctId, name->dbname);
return 0;
}

1
source/common/src/tvariant.c
View File

@ -16,7 +16,6 @@
#include "taos.h"
#include "taosdef.h"
#include "thash.h"
#include "ttime.h"
#include "ttokendef.h"
#include "ttypes.h"

4
source/dnode/bnode/src/bnode.c
View File

@ -15,7 +15,7 @@
#include "bndInt.h"
SBnode *bndOpen(const SBnodeOpt *pOption) {
SBnode *bndOpen(const char *path, const SBnodeOpt *pOption) {
SBnode *pBnode = calloc(1, sizeof(SBnode));
return pBnode;
}
@ -25,3 +25,5 @@ void bndClose(SBnode *pBnode) { free(pBnode); }
int32_t bndGetLoad(SBnode *pBnode, SBnodeLoad *pLoad) { return 0; }
int32_t bndProcessWMsgs(SBnode *pBnode, SArray *pMsgs) { return 0; }
void bndDestroy(const char *path) {}

2
source/dnode/mgmt/impl/inc/dndBnode.h
View File

@ -24,7 +24,7 @@ extern "C" {
int32_t dndInitBnode(SDnode *pDnode);
void dndCleanupBnode(SDnode *pDnode);
ioid dndProcessBnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
void dndProcessBnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);
int32_t dndProcessDropBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg);

76
source/dnode/mgmt/impl/inc/dndInt.h
View File

@ -54,8 +54,23 @@ extern int32_t dDebugFlag;
#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", dDebugFlag, __VA_ARGS__); }}
typedef enum { DND_STAT_INIT, DND_STAT_RUNNING, DND_STAT_STOPPED } EStat;
typedef enum { DND_WORKER_SINGLE, DND_WORKER_MULTI } EWorkerType;
typedef void (*DndMsgFp)(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEps);
typedef struct {
EWorkerType type;
const char *name;
int32_t minNum;
int32_t maxNum;
void *queueFp;
SDnode *pDnode;
taos_queue queue;
union {
SWorkerPool pool;
SMWorkerPool mpool;
};
} SDnodeWorker;
typedef struct {
char *dnode;
char *mnode;
@ -82,56 +97,45 @@ typedef struct {
} SDnodeMgmt;
typedef struct {
int32_t refCount;
int8_t deployed;
int8_t dropped;
int8_t replica;
int8_t selfIndex;
SReplica replicas[TSDB_MAX_REPLICA];
char *file;
SMnode *pMnode;
SRWLatch latch;
taos_queue pReadQ;
taos_queue pWriteQ;
taos_queue pSyncQ;
SWorkerPool readPool;
SWorkerPool writePool;
SWorkerPool syncPool;
int32_t refCount;
int8_t deployed;
int8_t dropped;
SMnode *pMnode;
SRWLatch latch;
SDnodeWorker readWorker;
SDnodeWorker writeWorker;
SDnodeWorker syncWorker;
int8_t replica;
int8_t selfIndex;
SReplica replicas[TSDB_MAX_REPLICA];
} SMnodeMgmt;
typedef struct {
int32_t refCount;
int8_t deployed;
int8_t dropped;
char *file;
SQnode *pQnode;
SRWLatch latch;
taos_queue pQueryQ;
taos_queue pFetchQ;
SWorkerPool queryPool;
SWorkerPool fetchPool;
int32_t refCount;
int8_t deployed;
int8_t dropped;
SQnode *pQnode;
SRWLatch latch;
SDnodeWorker queryWorker;
SDnodeWorker fetchWorker;
} SQnodeMgmt;
typedef struct {
int32_t refCount;
int8_t deployed;
int8_t dropped;
char *file;
SSnode *pSnode;
SRWLatch latch;
taos_queue pWriteQ;
SWorkerPool writePool;
int32_t refCount;
int8_t deployed;
int8_t dropped;
SSnode *pSnode;
SRWLatch latch;
SDnodeWorker writeWorker;
} SSnodeMgmt;
typedef struct {
int32_t refCount;
int8_t deployed;
int8_t dropped;
char *file;
SBnode *pBnode;
SRWLatch latch;
taos_queue pWriteQ;
SMWorkerPool writePool;
SDnodeWorker writeWorker;
} SBnodeMgmt;
typedef struct {

33
source/dnode/mgmt/impl/inc/dndWorker.h Normal file
View File

@ -0,0 +1,33 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef _TD_DND_WORKER_H_
#define _TD_DND_WORKER_H_
#ifdef __cplusplus
extern "C" {
#endif
#include "dndInt.h"
int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum,
int32_t maxNum, void *queueFp);
void dndCleanupWorker(SDnodeWorker *pWorker);
int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pCont, int32_t contLen);
#ifdef __cplusplus
}
#endif
#endif /*_TD_DND_WORKER_H_*/

379
source/dnode/mgmt/impl/src/dndBnode.c Normal file
View File

@ -0,0 +1,379 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http:www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "dndBnode.h"
#include "dndDnode.h"
#include "dndTransport.h"
#include "dndWorker.h"
static void dndProcessBnodeQueue(SDnode *pDnode, taos_qall qall, int32_t numOfMsgs);
static SBnode *dndAcquireBnode(SDnode *pDnode) {
SBnodeMgmt *pMgmt = &pDnode->bmgmt;
SBnode *pBnode = NULL;
int32_t refCount = 0;
taosRLockLatch(&pMgmt->latch);
if (pMgmt->deployed && !pMgmt->dropped) {
refCount = atomic_add_fetch_32(&pMgmt->refCount, 1);
pBnode = pMgmt->pBnode;
} else {
terrno = TSDB_CODE_DND_BNODE_NOT_DEPLOYED;
}
taosRUnLockLatch(&pMgmt->latch);
if (pBnode != NULL) {
dTrace("acquire bnode, refCount:%d", refCount);
}
return pBnode;
}
static void dndReleaseBnode(SDnode *pDnode, SBnode *pBnode) {
SBnodeMgmt *pMgmt = &pDnode->bmgmt;
int32_t refCount = 0;
taosRLockLatch(&pMgmt->latch);
if (pBnode != NULL) {
refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1);
}
taosRUnLockLatch(&pMgmt->latch);
if (pBnode != NULL) {
dTrace("release bnode, refCount:%d", refCount);
}
}
static int32_t dndReadBnodeFile(SDnode *pDnode) {
SBnodeMgmt *pMgmt = &pDnode->bmgmt;
int32_t code = TSDB_CODE_DND_BNODE_READ_FILE_ERROR;
int32_t len = 0;
int32_t maxLen = 4096;
char *content = calloc(1, maxLen + 1);
cJSON *root = NULL;
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "r");
if (fp == NULL) {
dDebug("file %s not exist", file);
code = 0;
goto PRASE_BNODE_OVER;
}
len = (int32_t)fread(content, 1, maxLen, fp);
if (len <= 0) {
dError("failed to read %s since content is null", file);
goto PRASE_BNODE_OVER;
}
content[len] = 0;
root = cJSON_Parse(content);
if (root == NULL) {
dError("failed to read %s since invalid json format", file);
goto PRASE_BNODE_OVER;
}
cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
if (!deployed || deployed->type != cJSON_Number) {
dError("failed to read %s since deployed not found", file);
goto PRASE_BNODE_OVER;
}
pMgmt->deployed = deployed->valueint;
cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
if (!dropped || dropped->type != cJSON_Number) {
dError("failed to read %s since dropped not found", file);
goto PRASE_BNODE_OVER;
}
pMgmt->dropped = dropped->valueint;
code = 0;
dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped);
PRASE_BNODE_OVER:
if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp);
terrno = code;
return code;
}
static int32_t dndWriteBnodeFile(SDnode *pDnode) {
SBnodeMgmt *pMgmt = &pDnode->bmgmt;
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "w");
if (fp == NULL) {
terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR;
dError("failed to write %s since %s", file, terrstr());
return -1;
}
int32_t len = 0;
int32_t maxLen = 4096;
char *content = calloc(1, maxLen + 1);
len += snprintf(content + len, maxLen - len, "{\n");
len += snprintf(content + len, maxLen - len, " \"deployed\": %d,\n", pMgmt->deployed);
len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped);
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
taosFsyncFile(fileno(fp));
fclose(fp);
free(content);
char realfile[PATH_MAX + 20];
snprintf(realfile, PATH_MAX + 20, "%s/bnode.json", pDnode->dir.dnode);
if (taosRenameFile(file, realfile) != 0) {
terrno = TSDB_CODE_DND_BNODE_WRITE_FILE_ERROR;
dError("failed to rename %s since %s", file, terrstr());
return -1;
}
dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped);
return 0;
}
static int32_t dndStartBnodeWorker(SDnode *pDnode) {
SBnodeMgmt *pMgmt = &pDnode->bmgmt;
if (dndInitWorker(pDnode, &pMgmt->writeWorker, DND_WORKER_MULTI, "bnode-write", 0, 1, dndProcessBnodeQueue) != 0) {
dError("failed to start bnode write worker since %s", terrstr());
return -1;
}
return 0;
}
static void dndStopBnodeWorker(SDnode *pDnode) {
SBnodeMgmt *pMgmt = &pDnode->bmgmt;
taosWLockLatch(&pMgmt->latch);
pMgmt->deployed = 0;
taosWUnLockLatch(&pMgmt->latch);
while (pMgmt->refCount > 1) {
taosMsleep(10);
}
dndCleanupWorker(&pMgmt->writeWorker);
}
static void dndBuildBnodeOption(SDnode *pDnode, SBnodeOpt *pOption) {
pOption->pDnode = pDnode;
pOption->sendMsgToDnodeFp = dndSendMsgToDnode;
pOption->sendMsgToMnodeFp = dndSendMsgToMnode;
pOption->sendRedirectMsgFp = dndSendRedirectMsg;
pOption->dnodeId = dndGetDnodeId(pDnode);
pOption->clusterId = dndGetClusterId(pDnode);
pOption->cfg.sver = pDnode->opt.sver;
}
static int32_t dndOpenBnode(SDnode *pDnode) {
SBnodeMgmt *pMgmt = &pDnode->bmgmt;
SBnodeOpt option = {0};
dndBuildBnodeOption(pDnode, &option);
SBnode *pBnode = bndOpen(pDnode->dir.bnode, &option);
if (pBnode == NULL) {
dError("failed to open bnode since %s", terrstr());
return -1;
}
if (dndStartBnodeWorker(pDnode) != 0) {
dError("failed to start bnode worker since %s", terrstr());
bndClose(pBnode);
return -1;
}
pMgmt->deployed = 1;
if (dndWriteBnodeFile(pDnode) != 0) {
pMgmt->deployed = 0;
dError("failed to write bnode file since %s", terrstr());
dndStopBnodeWorker(pDnode);
bndClose(pBnode);
return -1;
}
taosWLockLatch(&pMgmt->latch);
pMgmt->pBnode = pBnode;
taosWUnLockLatch(&pMgmt->latch);
dInfo("bnode open successfully");
return 0;
}
static int32_t dndDropBnode(SDnode *pDnode) {
SBnodeMgmt *pMgmt = &pDnode->bmgmt;
SBnode *pBnode = dndAcquireBnode(pDnode);
if (pBnode == NULL) {
dError("failed to drop bnode since %s", terrstr());
return -1;
}
taosRLockLatch(&pMgmt->latch);
pMgmt->dropped = 1;
taosRUnLockLatch(&pMgmt->latch);
if (dndWriteBnodeFile(pDnode) != 0) {
taosRLockLatch(&pMgmt->latch);
pMgmt->dropped = 0;
taosRUnLockLatch(&pMgmt->latch);
dndReleaseBnode(pDnode, pBnode);
dError("failed to drop bnode since %s", terrstr());
return -1;
}
dndReleaseBnode(pDnode, pBnode);
dndStopBnodeWorker(pDnode);
pMgmt->deployed = 0;
dndWriteBnodeFile(pDnode);
bndClose(pBnode);
pMgmt->pBnode = NULL;
bndDestroy(pDnode->dir.bnode);
return 0;
}
int32_t dndProcessCreateBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) {
SCreateBnodeInMsg *pMsg = pRpcMsg->pCont;
pMsg->dnodeId = htonl(pMsg->dnodeId);
if (pMsg->dnodeId != dndGetDnodeId(pDnode)) {
terrno = TSDB_CODE_DND_BNODE_ID_INVALID;
return -1;
} else {
return dndOpenBnode(pDnode);
}
}
int32_t dndProcessDropBnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) {
SDropBnodeInMsg *pMsg = pRpcMsg->pCont;
pMsg->dnodeId = htonl(pMsg->dnodeId);
if (pMsg->dnodeId != dndGetDnodeId(pDnode)) {
terrno = TSDB_CODE_DND_BNODE_ID_INVALID;
return -1;
} else {
return dndDropBnode(pDnode);
}
}
static void dndSendBnodeErrorRsp(SRpcMsg *pMsg, int32_t code) {
SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
rpcSendResponse(&rpcRsp);
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
}
static void dndSendBnodeErrorRsps(taos_qall qall, int32_t numOfMsgs, int32_t code) {
for (int32_t i = 0; i < numOfMsgs; ++i) {
SRpcMsg *pMsg = NULL;
taosGetQitem(qall, (void **)&pMsg);
dndSendBnodeErrorRsp(pMsg, code);
}
}
static void dndProcessBnodeQueue(SDnode *pDnode, taos_qall qall, int32_t numOfMsgs) {
SBnode *pBnode = dndAcquireBnode(pDnode);
if (pBnode == NULL) {
dndSendBnodeErrorRsps(qall, numOfMsgs, TSDB_CODE_OUT_OF_MEMORY);
return;
}
SArray *pArray = taosArrayInit(numOfMsgs, sizeof(SRpcMsg *));
if (pArray == NULL) {
dndReleaseBnode(pDnode, pBnode);
dndSendBnodeErrorRsps(qall, numOfMsgs, TSDB_CODE_OUT_OF_MEMORY);
return;
}
for (int32_t i = 0; i < numOfMsgs; ++i) {
SRpcMsg *pMsg = NULL;
taosGetQitem(qall, (void **)&pMsg);
void *ptr = taosArrayPush(pArray, &pMsg);
if (ptr == NULL) {
dndSendBnodeErrorRsp(pMsg, TSDB_CODE_OUT_OF_MEMORY);
}
}
bndProcessWMsgs(pBnode, pArray);
for (size_t i = 0; i < numOfMsgs; i++) {
SRpcMsg *pMsg = *(SRpcMsg **)taosArrayGet(pArray, i);
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
}
taosArrayDestroy(pArray);
dndReleaseBnode(pDnode, pBnode);
}
static void dndWriteBnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pMsg) {
int32_t code = TSDB_CODE_DND_BNODE_NOT_DEPLOYED;
SBnode *pBnode = dndAcquireBnode(pDnode);
if (pBnode != NULL) {
code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg));
}
dndReleaseBnode(pDnode, pBnode);
if (code != 0) {
if (pMsg->msgType & 1u) {
SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
rpcSendResponse(&rsp);
}
rpcFreeCont(pMsg->pCont);
}
}
void dndProcessBnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
dndWriteBnodeMsgToWorker(pDnode, &pDnode->bmgmt.writeWorker, pMsg);
}
int32_t dndInitBnode(SDnode *pDnode) {
SBnodeMgmt *pMgmt = &pDnode->bmgmt;
taosInitRWLatch(&pMgmt->latch);
if (dndReadBnodeFile(pDnode) != 0) {
return -1;
}
if (pMgmt->dropped) {
dInfo("bnode has been deployed and needs to be deleted");
bndDestroy(pDnode->dir.bnode);
return 0;
}
if (!pMgmt->deployed) return 0;
return dndOpenBnode(pDnode);
}
void dndCleanupBnode(SDnode *pDnode) {
SBnodeMgmt *pMgmt = &pDnode->bmgmt;
if (pMgmt->pBnode) {
dndStopBnodeWorker(pDnode);
bndClose(pMgmt->pBnode);
pMgmt->pBnode = NULL;
}
}

368
source/dnode/mgmt/impl/src/dndMnode.c
View File

@ -17,42 +17,9 @@
#include "dndMnode.h"
#include "dndDnode.h"
#include "dndTransport.h"
#include "dndWorker.h"
static int32_t dndInitMnodeReadWorker(SDnode *pDnode);
static int32_t dndInitMnodeWriteWorker(SDnode *pDnode);
static int32_t dndInitMnodeSyncWorker(SDnode *pDnode);
static void dndCleanupMnodeReadWorker(SDnode *pDnode);
static void dndCleanupMnodeWriteWorker(SDnode *pDnode);
static void dndCleanupMnodeSyncWorker(SDnode *pDnode);
static void dndCleanupMnodeMgmtWorker(SDnode *pDnode);
static int32_t dndAllocMnodeReadQueue(SDnode *pDnode);
static int32_t dndAllocMnodeWriteQueue(SDnode *pDnode);
static int32_t dndAllocMnodeSyncQueue(SDnode *pDnode);
static void dndFreeMnodeReadQueue(SDnode *pDnode);
static void dndFreeMnodeWriteQueue(SDnode *pDnode);
static void dndFreeMnodeSyncQueue(SDnode *pDnode);
static void dndFreeMnodeMgmtQueue(SDnode *pDnode);
static void dndProcessMnodeReadQueue(SDnode *pDnode, SMnodeMsg *pMsg);
static void dndProcessMnodeWriteQueue(SDnode *pDnode, SMnodeMsg *pMsg);
static void dndProcessMnodeSyncQueue(SDnode *pDnode, SMnodeMsg *pMsg);
static int32_t dndWriteMnodeMsgToQueue(SMnode *pMnode, taos_queue pQueue, SRpcMsg *pRpcMsg);
void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet);
static int32_t dndStartMnodeWorker(SDnode *pDnode);
static void dndStopMnodeWorker(SDnode *pDnode);
static SMnode *dndAcquireMnode(SDnode *pDnode);
static void dndReleaseMnode(SDnode *pDnode, SMnode *pMnode);
static int32_t dndReadMnodeFile(SDnode *pDnode);
static int32_t dndWriteMnodeFile(SDnode *pDnode);
static int32_t dndOpenMnode(SDnode *pDnode, SMnodeOpt *pOption);
static int32_t dndAlterMnode(SDnode *pDnode, SMnodeOpt *pOption);
static int32_t dndDropMnode(SDnode *pDnode);
static void dndProcessMnodeQueue(SDnode *pDnode, SMnodeMsg *pMsg);
static SMnode *dndAcquireMnode(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
@ -97,49 +64,52 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) {
char *content = calloc(1, maxLen + 1);
cJSON *root = NULL;
FILE *fp = fopen(pMgmt->file, "r");
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "r");
if (fp == NULL) {
dDebug("file %s not exist", pMgmt->file);
dDebug("file %s not exist", file);
code = 0;
goto PRASE_MNODE_OVER;
}
len = (int32_t)fread(content, 1, maxLen, fp);
if (len <= 0) {
dError("failed to read %s since content is null", pMgmt->file);
dError("failed to read %s since content is null", file);
goto PRASE_MNODE_OVER;
}
content[len] = 0;
root = cJSON_Parse(content);
if (root == NULL) {
dError("failed to read %s since invalid json format", pMgmt->file);
dError("failed to read %s since invalid json format", file);
goto PRASE_MNODE_OVER;
}
cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
if (!deployed || deployed->type != cJSON_Number) {
dError("failed to read %s since deployed not found", pMgmt->file);
dError("failed to read %s since deployed not found", file);
goto PRASE_MNODE_OVER;
}
pMgmt->deployed = deployed->valueint;
cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
if (!dropped || dropped->type != cJSON_Number) {
dError("failed to read %s since dropped not found", pMgmt->file);
dError("failed to read %s since dropped not found", file);
goto PRASE_MNODE_OVER;
}
pMgmt->dropped = dropped->valueint;
cJSON *mnodes = cJSON_GetObjectItem(root, "mnodes");
if (!mnodes || mnodes->type != cJSON_Array) {
dError("failed to read %s since nodes not found", pMgmt->file);
dError("failed to read %s since nodes not found", file);
goto PRASE_MNODE_OVER;
}
pMgmt->replica = cJSON_GetArraySize(mnodes);
if (pMgmt->replica <= 0 || pMgmt->replica > TSDB_MAX_REPLICA) {
dError("failed to read %s since mnodes size %d invalid", pMgmt->file, pMgmt->replica);
dError("failed to read %s since mnodes size %d invalid", file, pMgmt->replica);
goto PRASE_MNODE_OVER;
}
@ -151,28 +121,28 @@ static int32_t dndReadMnodeFile(SDnode *pDnode) {
cJSON *id = cJSON_GetObjectItem(node, "id");
if (!id || id->type != cJSON_Number) {
dError("failed to read %s since id not found", pMgmt->file);
dError("failed to read %s since id not found", file);
goto PRASE_MNODE_OVER;
}
pReplica->id = id->valueint;
cJSON *fqdn = cJSON_GetObjectItem(node, "fqdn");
if (!fqdn || fqdn->type != cJSON_String || fqdn->valuestring == NULL) {
dError("failed to read %s since fqdn not found", pMgmt->file);
dError("failed to read %s since fqdn not found", file);
goto PRASE_MNODE_OVER;
}
tstrncpy(pReplica->fqdn, fqdn->valuestring, TSDB_FQDN_LEN);
cJSON *port = cJSON_GetObjectItem(node, "port");
if (!port || port->type != cJSON_Number) {
dError("failed to read %s since port not found", pMgmt->file);
dError("failed to read %s since port not found", file);
goto PRASE_MNODE_OVER;
}
pReplica->port = port->valueint;
}
code = 0;
dDebug("succcessed to read file %s, deployed:%d dropped:%d", pMgmt->file, pMgmt->deployed, pMgmt->dropped);
dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped);
PRASE_MNODE_OVER:
if (content != NULL) free(content);
@ -186,8 +156,8 @@ PRASE_MNODE_OVER:
static int32_t dndWriteMnodeFile(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
char file[PATH_MAX + 20] = {0};
snprintf(file, sizeof(file), "%s.bak", pMgmt->file);
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/mnode.json.bak", pDnode->dir.dnode);
FILE *fp = fopen(file, "w");
if (fp == NULL) {
@ -223,47 +193,36 @@ static int32_t dndWriteMnodeFile(SDnode *pDnode) {
fclose(fp);
free(content);
if (taosRenameFile(file, pMgmt->file) != 0) {
char realfile[PATH_MAX + 20];
snprintf(realfile, PATH_MAX + 20, "%s/mnode.json", pDnode->dir.dnode);
if (taosRenameFile(file, realfile) != 0) {
terrno = TSDB_CODE_DND_MNODE_WRITE_FILE_ERROR;
dError("failed to rename %s since %s", pMgmt->file, terrstr());
dError("failed to rename %s since %s", file, terrstr());
return -1;
}
dInfo("successed to write %s, deployed:%d dropped:%d", pMgmt->file, pMgmt->deployed, pMgmt->dropped);
dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped);
return 0;
}
static int32_t dndStartMnodeWorker(SDnode *pDnode) {
if (dndInitMnodeReadWorker(pDnode) != 0) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
if (dndInitWorker(pDnode, &pMgmt->readWorker, DND_WORKER_SINGLE, "mnode-read", 0, 1, dndProcessMnodeQueue) != 0) {
dError("failed to start mnode read worker since %s", terrstr());
return -1;
}
if (dndInitMnodeWriteWorker(pDnode) != 0) {
if (dndInitWorker(pDnode, &pMgmt->writeWorker, DND_WORKER_SINGLE, "mnode-write", 0, 1, dndProcessMnodeQueue) != 0) {
dError("failed to start mnode write worker since %s", terrstr());
return -1;
}
if (dndInitMnodeSyncWorker(pDnode) != 0) {
if (dndInitWorker(pDnode, &pMgmt->syncWorker, DND_WORKER_SINGLE, "mnode-sync", 0, 1, dndProcessMnodeQueue) != 0) {
dError("failed to start mnode sync worker since %s", terrstr());
return -1;
}
if (dndAllocMnodeReadQueue(pDnode) != 0) {
dError("failed to alloc mnode read queue since %s", terrstr());
return -1;
}
if (dndAllocMnodeWriteQueue(pDnode) != 0) {
dError("failed to alloc mnode write queue since %s", terrstr());
return -1;
}
if (dndAllocMnodeSyncQueue(pDnode) != 0) {
dError("failed to alloc mnode sync queue since %s", terrstr());
return -1;
}
return 0;
}
@ -274,18 +233,13 @@ static void dndStopMnodeWorker(SDnode *pDnode) {
pMgmt->deployed = 0;
taosWUnLockLatch(&pMgmt->latch);
while (pMgmt->refCount > 1) taosMsleep(10);
while (!taosQueueEmpty(pMgmt->pReadQ)) taosMsleep(10);
while (!taosQueueEmpty(pMgmt->pWriteQ)) taosMsleep(10);
while (!taosQueueEmpty(pMgmt->pSyncQ)) taosMsleep(10);
while (pMgmt->refCount > 1) {
taosMsleep(10);
}
dndCleanupMnodeReadWorker(pDnode);
dndCleanupMnodeWriteWorker(pDnode);
dndCleanupMnodeSyncWorker(pDnode);
dndFreeMnodeReadQueue(pDnode);
dndFreeMnodeWriteQueue(pDnode);
dndFreeMnodeSyncQueue(pDnode);
dndCleanupWorker(&pMgmt->readWorker);
dndCleanupWorker(&pMgmt->writeWorker);
dndCleanupWorker(&pMgmt->syncWorker);
}
static bool dndNeedDeployMnode(SDnode *pDnode) {
@ -383,28 +337,21 @@ static int32_t dndOpenMnode(SDnode *pDnode, SMnodeOpt *pOption) {
dError("failed to open mnode since %s", terrstr());
return -1;
}
pMgmt->deployed = 1;
int32_t code = dndWriteMnodeFile(pDnode);
if (code != 0) {
dError("failed to write mnode file since %s", terrstr());
code = terrno;
pMgmt->deployed = 0;
if (dndStartMnodeWorker(pDnode) != 0) {
dError("failed to start mnode worker since %s", terrstr());
mndClose(pMnode);
mndDestroy(pDnode->dir.mnode);
terrno = code;
return -1;
}
code = dndStartMnodeWorker(pDnode);
if (code != 0) {
dError("failed to start mnode worker since %s", terrstr());
code = terrno;
pMgmt->deployed = 1;
if (dndWriteMnodeFile(pDnode) != 0) {
dError("failed to write mnode file since %s", terrstr());
pMgmt->deployed = 0;
dndStopMnodeWorker(pDnode);
mndClose(pMnode);
mndDestroy(pDnode->dir.mnode);
terrno = code;
return -1;
}
@ -461,6 +408,7 @@ static int32_t dndDropMnode(SDnode *pDnode) {
dndReleaseMnode(pDnode, pMnode);
dndStopMnodeWorker(pDnode);
pMgmt->deployed = 0;
dndWriteMnodeFile(pDnode);
mndClose(pMnode);
pMgmt->pMnode = NULL;
@ -528,13 +476,12 @@ int32_t dndProcessDropMnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) {
}
}
static void dndProcessMnodeReadQueue(SDnode *pDnode, SMnodeMsg *pMsg) {
static void dndProcessMnodeQueue(SDnode *pDnode, SMnodeMsg *pMsg) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
SMnode *pMnode = dndAcquireMnode(pDnode);
if (pMnode != NULL) {
mndProcessReadMsg(pMsg);
mndProcessMsg(pMsg);
dndReleaseMnode(pDnode, pMnode);
} else {
mndSendRsp(pMsg, terrno);
@ -543,208 +490,43 @@ static void dndProcessMnodeReadQueue(SDnode *pDnode, SMnodeMsg *pMsg) {
mndCleanupMsg(pMsg);
}
static void dndProcessMnodeWriteQueue(SDnode *pDnode, SMnodeMsg *pMsg) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
static void dndWriteMnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pRpcMsg) {
int32_t code = TSDB_CODE_DND_MNODE_NOT_DEPLOYED;
SMnode *pMnode = dndAcquireMnode(pDnode);
if (pMnode != NULL) {
mndProcessWriteMsg(pMsg);
dndReleaseMnode(pDnode, pMnode);
} else {
mndSendRsp(pMsg, terrno);
SMnodeMsg *pMsg = mndInitMsg(pMnode, pRpcMsg);
if (pMsg == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
} else {
code = dndWriteMsgToWorker(pWorker, pMsg, 0);
}
if (code != 0) {
mndCleanupMsg(pMsg);
}
}
dndReleaseMnode(pDnode, pMnode);
mndCleanupMsg(pMsg);
}
static void dndProcessMnodeSyncQueue(SDnode *pDnode, SMnodeMsg *pMsg) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
SMnode *pMnode = dndAcquireMnode(pDnode);
if (pMnode != NULL) {
mndProcessSyncMsg(pMsg);
dndReleaseMnode(pDnode, pMnode);
} else {
mndSendRsp(pMsg, terrno);
if (code != 0) {
if (pRpcMsg->msgType & 1u) {
SRpcMsg rsp = {.handle = pRpcMsg->handle, .ahandle = pRpcMsg->ahandle, .code = code};
rpcSendResponse(&rsp);
}
rpcFreeCont(pRpcMsg->pCont);
}
mndCleanupMsg(pMsg);
}
static int32_t dndWriteMnodeMsgToQueue(SMnode *pMnode, taos_queue pQueue, SRpcMsg *pRpcMsg) {
SMnodeMsg *pMsg = mndInitMsg(pMnode, pRpcMsg);
if (pMsg == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
if (taosWriteQitem(pQueue, pMsg) != 0) {
mndCleanupMsg(pMsg);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
return 0;
}
void dndProcessMnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
SMnode *pMnode = dndAcquireMnode(pDnode);
if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pWriteQ, pMsg) != 0) {
if (pMsg->msgType & 1u) {
SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno};
rpcSendResponse(&rsp);
}
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
}
dndReleaseMnode(pDnode, pMnode);
dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.writeWorker, pMsg);
}
void dndProcessMnodeSyncMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
SMnode *pMnode = dndAcquireMnode(pDnode);
if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pSyncQ, pMsg) != 0) {
if (pMsg->msgType & 1u) {
SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno};
rpcSendResponse(&rsp);
}
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
}
dndReleaseMnode(pDnode, pMnode);
dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.syncWorker, pMsg);
}
void dndProcessMnodeReadMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
SMnode *pMnode = dndAcquireMnode(pDnode);
if (pMnode == NULL || dndWriteMnodeMsgToQueue(pMnode, pMgmt->pReadQ, pMsg) != 0) {
if (pMsg->msgType & 1u) {
SRpcMsg rsp = {.handle = pMsg->handle, .code = terrno};
rpcSendResponse(&rsp);
}
rpcFreeCont(pMsg->pCont);
pMsg->pCont = NULL;
}
dndReleaseMnode(pDnode, pMnode);
}
static int32_t dndAllocMnodeReadQueue(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
pMgmt->pReadQ = tWorkerAllocQueue(&pMgmt->readPool, pDnode, (FProcessItem)dndProcessMnodeReadQueue);
if (pMgmt->pReadQ == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
return 0;
}
static void dndFreeMnodeReadQueue(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
tWorkerFreeQueue(&pMgmt->readPool, pMgmt->pReadQ);
pMgmt->pReadQ = NULL;
}
static int32_t dndInitMnodeReadWorker(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
SWorkerPool *pPool = &pMgmt->readPool;
pPool->name = "mnode-read";
pPool->min = 0;
pPool->max = 1;
if (tWorkerInit(pPool) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
dDebug("mnode read worker is initialized");
return 0;
}
static void dndCleanupMnodeReadWorker(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
tWorkerCleanup(&pMgmt->readPool);
dDebug("mnode read worker is closed");
}
static int32_t dndAllocMnodeWriteQueue(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
pMgmt->pWriteQ = tWorkerAllocQueue(&pMgmt->writePool, pDnode, (FProcessItem)dndProcessMnodeWriteQueue);
if (pMgmt->pWriteQ == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
return 0;
}
static void dndFreeMnodeWriteQueue(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
tWorkerFreeQueue(&pMgmt->writePool, pMgmt->pWriteQ);
pMgmt->pWriteQ = NULL;
}
static int32_t dndInitMnodeWriteWorker(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
SWorkerPool *pPool = &pMgmt->writePool;
pPool->name = "mnode-write";
pPool->min = 0;
pPool->max = 1;
if (tWorkerInit(pPool) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
dDebug("mnode write worker is initialized");
return 0;
}
static void dndCleanupMnodeWriteWorker(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
tWorkerCleanup(&pMgmt->writePool);
dDebug("mnode write worker is closed");
}
static int32_t dndAllocMnodeSyncQueue(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
pMgmt->pSyncQ = tWorkerAllocQueue(&pMgmt->syncPool, pDnode, (FProcessItem)dndProcessMnodeSyncQueue);
if (pMgmt->pSyncQ == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
return 0;
}
static void dndFreeMnodeSyncQueue(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
tWorkerFreeQueue(&pMgmt->syncPool, pMgmt->pSyncQ);
pMgmt->pSyncQ = NULL;
}
static int32_t dndInitMnodeSyncWorker(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
SWorkerPool *pPool = &pMgmt->syncPool;
pPool->name = "mnode-sync";
pPool->min = 0;
pPool->max = 1;
if (tWorkerInit(pPool) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
dDebug("mnode sync worker is initialized");
return 0;
}
static void dndCleanupMnodeSyncWorker(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
tWorkerCleanup(&pMgmt->syncPool);
dDebug("mnode sync worker is closed");
dndWriteMnodeMsgToWorker(pDnode, &pDnode->mmgmt.readWorker, pMsg);
}
int32_t dndInitMnode(SDnode *pDnode) {
@ -752,14 +534,6 @@ int32_t dndInitMnode(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
taosInitRWLatch(&pMgmt->latch);
char path[PATH_MAX];
snprintf(path, PATH_MAX, "%s/mnode.json", pDnode->dir.dnode);
pMgmt->file = strdup(path);
if (pMgmt->file == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
if (dndReadMnodeFile(pDnode) != 0) {
return -1;
}
@ -790,13 +564,13 @@ int32_t dndInitMnode(SDnode *pDnode) {
}
void dndCleanupMnode(SDnode *pDnode) {
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
dInfo("dnode-mnode start to clean up");
if (pMgmt->pMnode) dndStopMnodeWorker(pDnode);
tfree(pMgmt->file);
mndClose(pMgmt->pMnode);
pMgmt->pMnode = NULL;
SMnodeMgmt *pMgmt = &pDnode->mmgmt;
if (pMgmt->pMnode) {
dndStopMnodeWorker(pDnode);
mndClose(pMgmt->pMnode);
pMgmt->pMnode = NULL;
}
dInfo("dnode-mnode is cleaned up");
}

260
source/dnode/mgmt/impl/src/dndQnode.c
View File

@ -17,30 +17,9 @@
#include "dndQnode.h"
#include "dndDnode.h"
#include "dndTransport.h"
#include "dndWorker.h"
static int32_t dndInitQnodeQueryWorker(SDnode *pDnode);
static int32_t dndInitQnodeFetchWorker(SDnode *pDnode);
static void dndCleanupQnodeQueryWorker(SDnode *pDnode);
static void dndCleanupQnodeFetchWorker(SDnode *pDnode);
static int32_t dndAllocQnodeQueryQueue(SDnode *pDnode);
static int32_t dndAllocQnodeFetchQueue(SDnode *pDnode);
static void dndFreeQnodeQueryQueue(SDnode *pDnode);
static void dndFreeQnodeFetchQueue(SDnode *pDnode);
static void dndProcessQnodeQueue(SDnode *pDnode, SRpcMsg *pMsg);
static int32_t dndWriteQnodeMsgToQueue(SQnode *pQnode, taos_queue pQueue, SRpcMsg *pRpcMsg);
static int32_t dndStartQnodeWorker(SDnode *pDnode);
static void dndStopQnodeWorker(SDnode *pDnode);
static SQnode *dndAcquireQnode(SDnode *pDnode);
static void dndReleaseQnode(SDnode *pDnode, SQnode *pQnode);
static int32_t dndReadQnodeFile(SDnode *pDnode);
static int32_t dndWriteQnodeFile(SDnode *pDnode);
static int32_t dndOpenQnode(SDnode *pDnode);
static int32_t dndDropQnode(SDnode *pDnode);
static void dndProcessQnodeQueue(SDnode *pDnode, SRpcMsg *pMsg);
static SQnode *dndAcquireQnode(SDnode *pDnode) {
SQnodeMgmt *pMgmt = &pDnode->qmgmt;
@ -85,44 +64,47 @@ static int32_t dndReadQnodeFile(SDnode *pDnode) {
char *content = calloc(1, maxLen + 1);
cJSON *root = NULL;
FILE *fp = fopen(pMgmt->file, "r");
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "r");
if (fp == NULL) {
dDebug("file %s not exist", pMgmt->file);
dDebug("file %s not exist", file);
code = 0;
goto PRASE_MNODE_OVER;
goto PRASE_QNODE_OVER;
}
len = (int32_t)fread(content, 1, maxLen, fp);
if (len <= 0) {
dError("failed to read %s since content is null", pMgmt->file);
goto PRASE_MNODE_OVER;
dError("failed to read %s since content is null", file);
goto PRASE_QNODE_OVER;
}
content[len] = 0;
root = cJSON_Parse(content);
if (root == NULL) {
dError("failed to read %s since invalid json format", pMgmt->file);
goto PRASE_MNODE_OVER;
dError("failed to read %s since invalid json format", file);
goto PRASE_QNODE_OVER;
}
cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
if (!deployed || deployed->type != cJSON_Number) {
dError("failed to read %s since deployed not found", pMgmt->file);
goto PRASE_MNODE_OVER;
dError("failed to read %s since deployed not found", file);
goto PRASE_QNODE_OVER;
}
pMgmt->deployed = deployed->valueint;
cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
if (!dropped || dropped->type != cJSON_Number) {
dError("failed to read %s since dropped not found", pMgmt->file);
goto PRASE_MNODE_OVER;
dError("failed to read %s since dropped not found", file);
goto PRASE_QNODE_OVER;
}
pMgmt->dropped = dropped->valueint;
code = 0;
dDebug("succcessed to read file %s, deployed:%d dropped:%d", pMgmt->file, pMgmt->deployed, pMgmt->dropped);
dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped);
PRASE_MNODE_OVER:
PRASE_QNODE_OVER:
if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp);
@ -134,8 +116,8 @@ PRASE_MNODE_OVER:
static int32_t dndWriteQnodeFile(SDnode *pDnode) {
SQnodeMgmt *pMgmt = &pDnode->qmgmt;
char file[PATH_MAX + 20] = {0};
snprintf(file, sizeof(file), "%s.bak", pMgmt->file);
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "w");
if (fp == NULL) {
@ -154,41 +136,35 @@ static int32_t dndWriteQnodeFile(SDnode *pDnode) {
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
taosFfetchFile(fileno(fp));
taosFsyncFile(fileno(fp));
fclose(fp);
free(content);
if (taosRenameFile(file, pMgmt->file) != 0) {
char realfile[PATH_MAX + 20];
snprintf(realfile, PATH_MAX + 20, "%s/qnode.json", pDnode->dir.dnode);
if (taosRenameFile(file, realfile) != 0) {
terrno = TSDB_CODE_DND_QNODE_WRITE_FILE_ERROR;
dError("failed to rename %s since %s", pMgmt->file, terrstr());
dError("failed to rename %s since %s", file, terrstr());
return -1;
}
dInfo("successed to write %s, deployed:%d dropped:%d", pMgmt->file, pMgmt->deployed, pMgmt->dropped);
dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped);
return 0;
}
static int32_t dndStartQnodeWorker(SDnode *pDnode) {
if (dndInitQnodeQueryWorker(pDnode) != 0) {
SQnodeMgmt *pMgmt = &pDnode->qmgmt;
if (dndInitWorker(pDnode, &pMgmt->queryWorker, DND_WORKER_SINGLE, "qnode-query", 0, 1, dndProcessQnodeQueue) != 0) {
dError("failed to start qnode query worker since %s", terrstr());
return -1;
}
if (dndInitQnodeFetchWorker(pDnode) != 0) {
if (dndInitWorker(pDnode, &pMgmt->fetchWorker, DND_WORKER_SINGLE, "qnode-fetch", 0, 1, dndProcessQnodeQueue) != 0) {
dError("failed to start qnode fetch worker since %s", terrstr());
return -1;
}
if (dndAllocQnodeQueryQueue(pDnode) != 0) {
dError("failed to alloc qnode query queue since %s", terrstr());
return -1;
}
if (dndAllocQnodeFetchQueue(pDnode) != 0) {
dError("failed to alloc qnode fetch queue since %s", terrstr());
return -1;
}
return 0;
}
@ -199,15 +175,12 @@ static void dndStopQnodeWorker(SDnode *pDnode) {
pMgmt->deployed = 0;
taosWUnLockLatch(&pMgmt->latch);
while (pMgmt->refCount > 1) taosMsleep(10);
while (!taosQueueEmpty(pMgmt->pQueryQ)) taosMsleep(10);
while (!taosQueueEmpty(pMgmt->pFetchQ)) taosMsleep(10);
while (pMgmt->refCount > 1) {
taosMsleep(10);
}
dndCleanupQnodeQueryWorker(pDnode);
dndCleanupQnodeFetchWorker(pDnode);
dndFreeQnodeQueryQueue(pDnode);
dndFreeQnodeFetchQueue(pDnode);
dndCleanupWorker(&pMgmt->queryWorker);
dndCleanupWorker(&pMgmt->fetchWorker);
}
static void dndBuildQnodeOption(SDnode *pDnode, SQnodeOpt *pOption) {
@ -230,34 +203,24 @@ static int32_t dndOpenQnode(SDnode *pDnode) {
dError("failed to open qnode since %s", terrstr());
return -1;
}
pMgmt->deployed = 1;
int32_t code = dndWriteQnodeFile(pDnode);
if (code != 0) {
dError("failed to write qnode file since %s", terrstr());
code = terrno;
pMgmt->deployed = 0;
if (dndStartQnodeWorker(pDnode) != 0) {
dError("failed to start qnode worker since %s", terrstr());
qndClose(pQnode);
// qndDestroy(pDnode->dir.qnode);
terrno = code;
return -1;
}
code = dndStartQnodeWorker(pDnode);
if (code != 0) {
dError("failed to start qnode worker since %s", terrstr());
code = terrno;
pMgmt->deployed = 1;
if (dndWriteQnodeFile(pDnode) != 0) {
pMgmt->deployed = 0;
dError("failed to write qnode file since %s", terrstr());
dndStopQnodeWorker(pDnode);
qndClose(pQnode);
// qndDestroy(pDnode->dir.qnode);
terrno = code;
return -1;
}
taosWLockLatch(&pMgmt->latch);
pMgmt->pQnode = pQnode;
pMgmt->deployed = 1;
taosWUnLockLatch(&pMgmt->latch);
dInfo("qnode open successfully");
@ -289,10 +252,10 @@ static int32_t dndDropQnode(SDnode *pDnode) {
dndReleaseQnode(pDnode, pQnode);
dndStopQnodeWorker(pDnode);
pMgmt->deployed = 0;
dndWriteQnodeFile(pDnode);
qndClose(pQnode);
pMgmt->pQnode = NULL;
// qndDestroy(pDnode->dir.qnode);
return 0;
}
@ -324,13 +287,11 @@ int32_t dndProcessDropQnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) {
static void dndProcessQnodeQueue(SDnode *pDnode, SRpcMsg *pMsg) {
SQnodeMgmt *pMgmt = &pDnode->qmgmt;
SRpcMsg *pRsp = NULL;
int32_t code = 0;
int32_t code = TSDB_CODE_DND_QNODE_NOT_DEPLOYED;
SQnode *pQnode = dndAcquireQnode(pDnode);
if (pQnode == NULL) {
code = -1;
} else {
code = qndProcessQueryReq(pQnode, pMsg, &pRsp);
if (pQnode != NULL) {
code = qndProcessMsg(pQnode, pMsg, &pRsp);
}
if (pRsp != NULL) {
@ -347,135 +308,36 @@ static void dndProcessQnodeQueue(SDnode *pDnode, SRpcMsg *pMsg) {
taosFreeQitem(pMsg);
}
static int32_t dndWriteQnodeMsgToQueue(SQnode *pQnode, taos_queue pQueue, SRpcMsg *pRpcMsg) {
int32_t code = 0;
static void dndWriteQnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pMsg) {
int32_t code = TSDB_CODE_DND_QNODE_NOT_DEPLOYED;
if (pQnode == NULL || pQueue == NULL) {
code = TSDB_CODE_DND_QNODE_NOT_DEPLOYED;
} else {
SRpcMsg *pMsg = taosAllocateQitem(sizeof(SRpcMsg));
if (pMsg == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
} else {
*pMsg = *pRpcMsg;
if (taosWriteQitem(pQueue, pMsg) != 0) {
code = TSDB_CODE_OUT_OF_MEMORY;
}
}
SQnode *pQnode = dndAcquireQnode(pDnode);
if (pQnode != NULL) {
code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg));
}
dndReleaseQnode(pDnode, pQnode);
if (code != 0) {
if (pRpcMsg->msgType & 1u) {
SRpcMsg rsp = {.handle = pRpcMsg->handle, .code = code};
if (pMsg->msgType & 1u) {
SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
rpcSendResponse(&rsp);
}
rpcFreeCont(pRpcMsg->pCont);
rpcFreeCont(pMsg->pCont);
}
}
void dndProcessQnodeQueryMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
SQnodeMgmt *pMgmt = &pDnode->qmgmt;
SQnode *pQnode = dndAcquireQnode(pDnode);
dndWriteQnodeMsgToQueue(pQnode, pMgmt->pQueryQ, pMsg);
dndReleaseQnode(pDnode, pQnode);
dndWriteQnodeMsgToWorker(pDnode, &pDnode->qmgmt.queryWorker, pMsg);
}
void dndProcessQnodeFetchMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
SQnodeMgmt *pMgmt = &pDnode->qmgmt;
SQnode *pQnode = dndAcquireQnode(pDnode);
dndWriteQnodeMsgToQueue(pQnode, pMgmt->pFetchQ, pMsg);
dndReleaseQnode(pDnode, pQnode);
}
static int32_t dndAllocQnodeQueryQueue(SDnode *pDnode) {
SQnodeMgmt *pMgmt = &pDnode->qmgmt;
pMgmt->pQueryQ = tWorkerAllocQueue(&pMgmt->queryPool, pDnode, (FProcessItem)dndProcessQnodeQueue);
if (pMgmt->pQueryQ == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
return 0;
}
static void dndFreeQnodeQueryQueue(SDnode *pDnode) {
SQnodeMgmt *pMgmt = &pDnode->qmgmt;
tWorkerFreeQueue(&pMgmt->queryPool, pMgmt->pQueryQ);
pMgmt->pQueryQ = NULL;
}
static int32_t dndInitQnodeQueryWorker(SDnode *pDnode) {
SQnodeMgmt *pMgmt = &pDnode->qmgmt;
SWorkerPool *pPool = &pMgmt->queryPool;
pPool->name = "qnode-query";
pPool->min = 0;
pPool->max = 1;
if (tWorkerInit(pPool) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
dDebug("qnode query worker is initialized");
return 0;
}
static void dndCleanupQnodeQueryWorker(SDnode *pDnode) {
SQnodeMgmt *pMgmt = &pDnode->qmgmt;
tWorkerCleanup(&pMgmt->queryPool);
dDebug("qnode query worker is closed");
}
static int32_t dndAllocQnodeFetchQueue(SDnode *pDnode) {
SQnodeMgmt *pMgmt = &pDnode->qmgmt;
pMgmt->pFetchQ = tWorkerAllocQueue(&pMgmt->queryPool, pDnode, (FProcessItem)dndProcessQnodeQueue);
if (pMgmt->pFetchQ == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
return 0;
}
static void dndFreeQnodeFetchQueue(SDnode *pDnode) {
SQnodeMgmt *pMgmt = &pDnode->qmgmt;
tWorkerFreeQueue(&pMgmt->fetchPool, pMgmt->pFetchQ);
pMgmt->pFetchQ = NULL;
}
static int32_t dndInitQnodeFetchWorker(SDnode *pDnode) {
SQnodeMgmt *pMgmt = &pDnode->qmgmt;
SWorkerPool *pPool = &pMgmt->fetchPool;
pPool->name = "qnode-fetch";
pPool->min = 0;
pPool->max = 1;
if (tWorkerInit(pPool) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
dDebug("qnode fetch worker is initialized");
return 0;
}
static void dndCleanupQnodeFetchWorker(SDnode *pDnode) {
SQnodeMgmt *pMgmt = &pDnode->qmgmt;
tWorkerCleanup(&pMgmt->fetchPool);
dDebug("qnode fetch worker is closed");
dndWriteQnodeMsgToWorker(pDnode, &pDnode->qmgmt.queryWorker, pMsg);
}
int32_t dndInitQnode(SDnode *pDnode) {
dInfo("dnode-qnode start to init");
SQnodeMgmt *pMgmt = &pDnode->qmgmt;
taosInitRWLatch(&pMgmt->latch);
char path[PATH_MAX];
snprintf(path, PATH_MAX, "%s/qnode.json", pDnode->dir.dnode);
pMgmt->file = strdup(path);
if (pMgmt->file == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
if (dndReadQnodeFile(pDnode) != 0) {
return -1;
}
@ -488,11 +350,9 @@ int32_t dndInitQnode(SDnode *pDnode) {
void dndCleanupQnode(SDnode *pDnode) {
SQnodeMgmt *pMgmt = &pDnode->qmgmt;
dInfo("dnode-qnode start to clean up");
if (pMgmt->pQnode) dndStopQnodeWorker(pDnode);
tfree(pMgmt->file);
qndClose(pMgmt->pQnode);
pMgmt->pQnode = NULL;
dInfo("dnode-qnode is cleaned up");
if (pMgmt->pQnode) {
dndStopQnodeWorker(pDnode);
qndClose(pMgmt->pQnode);
pMgmt->pQnode = NULL;
}
}

354
source/dnode/mgmt/impl/src/dndSnode.c Normal file
View File

@ -0,0 +1,354 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http:www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "dndSnode.h"
#include "dndDnode.h"
#include "dndTransport.h"
#include "dndWorker.h"
static void dndProcessSnodeQueue(SDnode *pDnode, SRpcMsg *pMsg);
static SSnode *dndAcquireSnode(SDnode *pDnode) {
SSnodeMgmt *pMgmt = &pDnode->smgmt;
SSnode *pSnode = NULL;
int32_t refCount = 0;
taosRLockLatch(&pMgmt->latch);
if (pMgmt->deployed && !pMgmt->dropped) {
refCount = atomic_add_fetch_32(&pMgmt->refCount, 1);
pSnode = pMgmt->pSnode;
} else {
terrno = TSDB_CODE_DND_SNODE_NOT_DEPLOYED;
}
taosRUnLockLatch(&pMgmt->latch);
if (pSnode != NULL) {
dTrace("acquire snode, refCount:%d", refCount);
}
return pSnode;
}
static void dndReleaseSnode(SDnode *pDnode, SSnode *pSnode) {
SSnodeMgmt *pMgmt = &pDnode->smgmt;
int32_t refCount = 0;
taosRLockLatch(&pMgmt->latch);
if (pSnode != NULL) {
refCount = atomic_sub_fetch_32(&pMgmt->refCount, 1);
}
taosRUnLockLatch(&pMgmt->latch);
if (pSnode != NULL) {
dTrace("release snode, refCount:%d", refCount);
}
}
static int32_t dndReadSnodeFile(SDnode *pDnode) {
SSnodeMgmt *pMgmt = &pDnode->smgmt;
int32_t code = TSDB_CODE_DND_SNODE_READ_FILE_ERROR;
int32_t len = 0;
int32_t maxLen = 4096;
char *content = calloc(1, maxLen + 1);
cJSON *root = NULL;
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "r");
if (fp == NULL) {
dDebug("file %s not exist", file);
code = 0;
goto PRASE_SNODE_OVER;
}
len = (int32_t)fread(content, 1, maxLen, fp);
if (len <= 0) {
dError("failed to read %s since content is null", file);
goto PRASE_SNODE_OVER;
}
content[len] = 0;
root = cJSON_Parse(content);
if (root == NULL) {
dError("failed to read %s since invalid json format", file);
goto PRASE_SNODE_OVER;
}
cJSON *deployed = cJSON_GetObjectItem(root, "deployed");
if (!deployed || deployed->type != cJSON_Number) {
dError("failed to read %s since deployed not found", file);
goto PRASE_SNODE_OVER;
}
pMgmt->deployed = deployed->valueint;
cJSON *dropped = cJSON_GetObjectItem(root, "dropped");
if (!dropped || dropped->type != cJSON_Number) {
dError("failed to read %s since dropped not found", file);
goto PRASE_SNODE_OVER;
}
pMgmt->dropped = dropped->valueint;
code = 0;
dDebug("succcessed to read file %s, deployed:%d dropped:%d", file, pMgmt->deployed, pMgmt->dropped);
PRASE_SNODE_OVER:
if (content != NULL) free(content);
if (root != NULL) cJSON_Delete(root);
if (fp != NULL) fclose(fp);
terrno = code;
return code;
}
static int32_t dndWriteSnodeFile(SDnode *pDnode) {
SSnodeMgmt *pMgmt = &pDnode->smgmt;
char file[PATH_MAX + 20];
snprintf(file, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode);
FILE *fp = fopen(file, "w");
if (fp == NULL) {
terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR;
dError("failed to write %s since %s", file, terrstr());
return -1;
}
int32_t len = 0;
int32_t maxLen = 4096;
char *content = calloc(1, maxLen + 1);
len += snprintf(content + len, maxLen - len, "{\n");
len += snprintf(content + len, maxLen - len, " \"deployed\": %d,\n", pMgmt->deployed);
len += snprintf(content + len, maxLen - len, " \"dropped\": %d\n", pMgmt->dropped);
len += snprintf(content + len, maxLen - len, "}\n");
fwrite(content, 1, len, fp);
taosFsyncFile(fileno(fp));
fclose(fp);
free(content);
char realfile[PATH_MAX + 20];
snprintf(realfile, PATH_MAX + 20, "%s/snode.json", pDnode->dir.dnode);
if (taosRenameFile(file, realfile) != 0) {
terrno = TSDB_CODE_DND_SNODE_WRITE_FILE_ERROR;
dError("failed to rename %s since %s", file, terrstr());
return -1;
}
dInfo("successed to write %s, deployed:%d dropped:%d", realfile, pMgmt->deployed, pMgmt->dropped);
return 0;
}
static int32_t dndStartSnodeWorker(SDnode *pDnode) {
SSnodeMgmt *pMgmt = &pDnode->smgmt;
if (dndInitWorker(pDnode, &pMgmt->writeWorker, DND_WORKER_SINGLE, "snode-write", 0, 1, dndProcessSnodeQueue) != 0) {
dError("failed to start snode write worker since %s", terrstr());
return -1;
}
return 0;
}
static void dndStopSnodeWorker(SDnode *pDnode) {
SSnodeMgmt *pMgmt = &pDnode->smgmt;
taosWLockLatch(&pMgmt->latch);
pMgmt->deployed = 0;
taosWUnLockLatch(&pMgmt->latch);
while (pMgmt->refCount > 1) {
taosMsleep(10);
}
dndCleanupWorker(&pMgmt->writeWorker);
}
static void dndBuildSnodeOption(SDnode *pDnode, SSnodeOpt *pOption) {
pOption->pDnode = pDnode;
pOption->sendMsgToDnodeFp = dndSendMsgToDnode;
pOption->sendMsgToMnodeFp = dndSendMsgToMnode;
pOption->sendRedirectMsgFp = dndSendRedirectMsg;
pOption->dnodeId = dndGetDnodeId(pDnode);
pOption->clusterId = dndGetClusterId(pDnode);
pOption->cfg.sver = pDnode->opt.sver;
}
static int32_t dndOpenSnode(SDnode *pDnode) {
SSnodeMgmt *pMgmt = &pDnode->smgmt;
SSnodeOpt option = {0};
dndBuildSnodeOption(pDnode, &option);
SSnode *pSnode = sndOpen(pDnode->dir.snode, &option);
if (pSnode == NULL) {
dError("failed to open snode since %s", terrstr());
return -1;
}
if (dndStartSnodeWorker(pDnode) != 0) {
dError("failed to start snode worker since %s", terrstr());
sndClose(pSnode);
return -1;
}
pMgmt->deployed = 1;
if (dndWriteSnodeFile(pDnode) != 0) {
pMgmt->deployed = 0;
dError("failed to write snode file since %s", terrstr());
dndStopSnodeWorker(pDnode);
sndClose(pSnode);
return -1;
}
taosWLockLatch(&pMgmt->latch);
pMgmt->pSnode = pSnode;
taosWUnLockLatch(&pMgmt->latch);
dInfo("snode open successfully");
return 0;
}
static int32_t dndDropSnode(SDnode *pDnode) {
SSnodeMgmt *pMgmt = &pDnode->smgmt;
SSnode *pSnode = dndAcquireSnode(pDnode);
if (pSnode == NULL) {
dError("failed to drop snode since %s", terrstr());
return -1;
}
taosRLockLatch(&pMgmt->latch);
pMgmt->dropped = 1;
taosRUnLockLatch(&pMgmt->latch);
if (dndWriteSnodeFile(pDnode) != 0) {
taosRLockLatch(&pMgmt->latch);
pMgmt->dropped = 0;
taosRUnLockLatch(&pMgmt->latch);
dndReleaseSnode(pDnode, pSnode);
dError("failed to drop snode since %s", terrstr());
return -1;
}
dndReleaseSnode(pDnode, pSnode);
dndStopSnodeWorker(pDnode);
pMgmt->deployed = 0;
dndWriteSnodeFile(pDnode);
sndClose(pSnode);
pMgmt->pSnode = NULL;
sndDestroy(pDnode->dir.snode);
return 0;
}
int32_t dndProcessCreateSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) {
SCreateSnodeInMsg *pMsg = pRpcMsg->pCont;
pMsg->dnodeId = htonl(pMsg->dnodeId);
if (pMsg->dnodeId != dndGetDnodeId(pDnode)) {
terrno = TSDB_CODE_DND_SNODE_ID_INVALID;
return -1;
} else {
return dndOpenSnode(pDnode);
}
}
int32_t dndProcessDropSnodeReq(SDnode *pDnode, SRpcMsg *pRpcMsg) {
SDropSnodeInMsg *pMsg = pRpcMsg->pCont;
pMsg->dnodeId = htonl(pMsg->dnodeId);
if (pMsg->dnodeId != dndGetDnodeId(pDnode)) {
terrno = TSDB_CODE_DND_SNODE_ID_INVALID;
return -1;
} else {
return dndDropSnode(pDnode);
}
}
static void dndProcessSnodeQueue(SDnode *pDnode, SRpcMsg *pMsg) {
SSnodeMgmt *pMgmt = &pDnode->smgmt;
SRpcMsg *pRsp = NULL;
int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED;
SSnode *pSnode = dndAcquireSnode(pDnode);
if (pSnode != NULL) {
code = sndProcessMsg(pSnode, pMsg, &pRsp);
}
if (pRsp != NULL) {
pRsp->ahandle = pMsg->ahandle;
rpcSendResponse(pRsp);
free(pRsp);
} else {
if (code != 0) code = terrno;
SRpcMsg rpcRsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
rpcSendResponse(&rpcRsp);
}
rpcFreeCont(pMsg->pCont);
taosFreeQitem(pMsg);
}
static void dndWriteSnodeMsgToWorker(SDnode *pDnode, SDnodeWorker *pWorker, SRpcMsg *pMsg) {
int32_t code = TSDB_CODE_DND_SNODE_NOT_DEPLOYED;
SSnode *pSnode = dndAcquireSnode(pDnode);
if (pSnode != NULL) {
code = dndWriteMsgToWorker(pWorker, pMsg, sizeof(SRpcMsg));
}
dndReleaseSnode(pDnode, pSnode);
if (code != 0) {
if (pMsg->msgType & 1u) {
SRpcMsg rsp = {.handle = pMsg->handle, .ahandle = pMsg->ahandle, .code = code};
rpcSendResponse(&rsp);
}
rpcFreeCont(pMsg->pCont);
}
}
void dndProcessSnodeWriteMsg(SDnode *pDnode, SRpcMsg *pMsg, SEpSet *pEpSet) {
dndWriteSnodeMsgToWorker(pDnode, &pDnode->smgmt.writeWorker, pMsg);
}
int32_t dndInitSnode(SDnode *pDnode) {
SSnodeMgmt *pMgmt = &pDnode->smgmt;
taosInitRWLatch(&pMgmt->latch);
if (dndReadSnodeFile(pDnode) != 0) {
return -1;
}
if (pMgmt->dropped) {
dInfo("snode has been deployed and needs to be deleted");
sndDestroy(pDnode->dir.snode);
return 0;
}
if (!pMgmt->deployed) return 0;
return dndOpenSnode(pDnode);
}
void dndCleanupSnode(SDnode *pDnode) {
SSnodeMgmt *pMgmt = &pDnode->smgmt;
if (pMgmt->pSnode) {
dndStopSnodeWorker(pDnode);
sndClose(pMgmt->pSnode);
pMgmt->pSnode = NULL;
}
}

6
source/dnode/mgmt/impl/src/dndVnodes.c
View File

@ -22,7 +22,7 @@ typedef struct {
int32_t vgVersion;
int8_t dropped;
uint64_t dbUid;
char db[TSDB_FULL_DB_NAME_LEN];
char db[TSDB_DB_FNAME_LEN];
char path[PATH_MAX + 20];
} SWrapperCfg;
@ -319,7 +319,7 @@ static int32_t dndGetVnodesFromFile(SDnode *pDnode, SWrapperCfg **ppCfgs, int32_
dError("failed to read %s since db not found", file);
goto PRASE_VNODE_OVER;
}
tstrncpy(pCfg->db, db->valuestring, TSDB_FULL_DB_NAME_LEN);
tstrncpy(pCfg->db, db->valuestring, TSDB_DB_FNAME_LEN);
}
*ppCfgs = pCfgs;
@ -569,7 +569,7 @@ static void dndGenerateVnodeCfg(SCreateVnodeMsg *pCreate, SVnodeCfg *pCfg) {
}
static void dndGenerateWrapperCfg(SDnode *pDnode, SCreateVnodeMsg *pCreate, SWrapperCfg *pCfg) {
memcpy(pCfg->db, pCreate->db, TSDB_FULL_DB_NAME_LEN);
memcpy(pCfg->db, pCreate->db, TSDB_DB_FNAME_LEN);
pCfg->dbUid = pCreate->dbUid;
pCfg->dropped = 0;
snprintf(pCfg->path, sizeof(pCfg->path), "%s/vnode%d", pDnode->dir.vnodes, pCreate->vgId);

110
source/dnode/mgmt/impl/src/dndWorker.c Normal file
View File

@ -0,0 +1,110 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http:www.gnu.org/licenses/>.
*/
#define _DEFAULT_SOURCE
#include "dndWorker.h"
int32_t dndInitWorker(SDnode *pDnode, SDnodeWorker *pWorker, EWorkerType type, const char *name, int32_t minNum,
int32_t maxNum, void *queueFp) {
if (pDnode == NULL || pWorker == NULL || name == NULL || minNum < 0 || maxNum <= 0 || queueFp == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return -1;
}
pWorker->type = type;
pWorker->name = name;
pWorker->minNum = minNum;
pWorker->maxNum = maxNum;
pWorker->queueFp = queueFp;
pWorker->pDnode = pDnode;
if (pWorker->type == DND_WORKER_SINGLE) {
SWorkerPool *pPool = &pWorker->pool;
pPool->name = name;
pPool->min = minNum;
pPool->max = maxNum;
if (tWorkerInit(pPool) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pWorker->queue = tWorkerAllocQueue(pPool, pDnode, (FProcessItem)queueFp);
if (pWorker->queue == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
} else if (pWorker->type == DND_WORKER_MULTI) {
SMWorkerPool *pPool = &pWorker->mpool;
pPool->name = name;
pPool->max = maxNum;
if (tMWorkerInit(pPool) != 0) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
pWorker->queue = tMWorkerAllocQueue(pPool, pDnode, (FProcessItems)queueFp);
if (pWorker->queue == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
} else {
terrno = TSDB_CODE_INVALID_PARA;
}
return 0;
}
void dndCleanupWorker(SDnodeWorker *pWorker) {
while (!taosQueueEmpty(pWorker->queue)) {
taosMsleep(10);
}
if (pWorker->type == DND_WORKER_SINGLE) {
tWorkerCleanup(&pWorker->pool);
tWorkerFreeQueue(&pWorker->pool, pWorker->queue);
} else if (pWorker->type == DND_WORKER_MULTI) {
tMWorkerCleanup(&pWorker->mpool);
tMWorkerFreeQueue(&pWorker->mpool, pWorker->queue);
} else {
}
}
int32_t dndWriteMsgToWorker(SDnodeWorker *pWorker, void *pCont, int32_t contLen) {
if (pWorker == NULL || pWorker->queue == NULL) {
terrno = TSDB_CODE_INVALID_PARA;
return -1;
}
void *pMsg = NULL;
if (contLen != 0) {
pMsg = taosAllocateQitem(contLen);
if (pMsg != NULL) {
memcpy(pMsg, pCont, contLen);
}
} else {
pMsg = pCont;
}
if (pMsg == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
if (taosWriteQitem(pWorker->queue, pMsg) != 0) {
taosFreeQitem(pMsg);
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
return 0;
}

12
source/dnode/mgmt/impl/test/dnode/dnode.cpp
View File

@ -91,7 +91,8 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) {
int32_t contLen = sizeof(SCreateDnodeMsg);
SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen);
strcpy(pReq->ep, "localhost:9042");
strcpy(pReq->fqdn, "localhost");
pReq->port = htonl(9042);
SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen);
ASSERT_NE(pMsg, nullptr);
@ -148,7 +149,8 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) {
int32_t contLen = sizeof(SCreateDnodeMsg);
SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen);
strcpy(pReq->ep, "localhost:9043");
strcpy(pReq->fqdn, "localhost");
pReq->port = htonl(9043);
SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen);
ASSERT_NE(pMsg, nullptr);
@ -159,7 +161,8 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) {
int32_t contLen = sizeof(SCreateDnodeMsg);
SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen);
strcpy(pReq->ep, "localhost:9044");
strcpy(pReq->fqdn, "localhost");
pReq->port = htonl(9044);
SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen);
ASSERT_NE(pMsg, nullptr);
@ -170,7 +173,8 @@ TEST_F(DndTestDnode, 03_Create_Drop_Restart_Dnode) {
int32_t contLen = sizeof(SCreateDnodeMsg);
SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen);
strcpy(pReq->ep, "localhost:9045");
strcpy(pReq->fqdn, "localhost");
pReq->port = htonl(9045);
SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen);
ASSERT_NE(pMsg, nullptr);

3
source/dnode/mgmt/impl/test/mnode/mnode.cpp
View File

@ -102,7 +102,8 @@ TEST_F(DndTestMnode, 04_Create_Mnode) {
int32_t contLen = sizeof(SCreateDnodeMsg);
SCreateDnodeMsg* pReq = (SCreateDnodeMsg*)rpcMallocCont(contLen);
strcpy(pReq->ep, "localhost:9062");
strcpy(pReq->fqdn, "localhost");
pReq->port = htonl(9062);
SRpcMsg* pMsg = test.SendMsg(TDMT_MND_CREATE_DNODE, pReq, contLen);
ASSERT_NE(pMsg, nullptr);

39
source/dnode/mgmt/impl/test/stb/stb.cpp
View File

@ -68,7 +68,6 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
{
SSchema* pSchema = &pReq->pSchema[0];
pSchema->colId = htonl(0);
pSchema->bytes = htonl(8);
pSchema->type = TSDB_DATA_TYPE_TIMESTAMP;
strcpy(pSchema->name, "ts");
@ -76,7 +75,6 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
{
SSchema* pSchema = &pReq->pSchema[1];
pSchema->colId = htonl(1);
pSchema->bytes = htonl(4);
pSchema->type = TSDB_DATA_TYPE_INT;
strcpy(pSchema->name, "col1");
@ -84,7 +82,6 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
{
SSchema* pSchema = &pReq->pSchema[2];
pSchema->colId = htonl(2);
pSchema->bytes = htonl(2);
pSchema->type = TSDB_DATA_TYPE_TINYINT;
strcpy(pSchema->name, "tag1");
@ -92,7 +89,6 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
{
SSchema* pSchema = &pReq->pSchema[3];
pSchema->colId = htonl(3);
pSchema->bytes = htonl(8);
pSchema->type = TSDB_DATA_TYPE_BIGINT;
strcpy(pSchema->name, "tag2");
@ -100,7 +96,6 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
{
SSchema* pSchema = &pReq->pSchema[4];
pSchema->colId = htonl(4);
pSchema->bytes = htonl(16);
pSchema->type = TSDB_DATA_TYPE_BINARY;
strcpy(pSchema->name, "tag3");
@ -167,10 +162,42 @@ TEST_F(DndTestStb, 01_Create_Show_Meta_Drop_Restart_Stb) {
{
SSchema* pSchema = &pRsp->pSchema[0];
EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TIMESTAMP);
EXPECT_EQ(pSchema->colId, 0);
EXPECT_EQ(pSchema->colId, 1);
EXPECT_EQ(pSchema->bytes, 8);
EXPECT_STREQ(pSchema->name, "ts");
}
{
SSchema* pSchema = &pRsp->pSchema[1];
EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_INT);
EXPECT_EQ(pSchema->colId, 2);
EXPECT_EQ(pSchema->bytes, 4);
EXPECT_STREQ(pSchema->name, "col1");
}
{
SSchema* pSchema = &pRsp->pSchema[2];
EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_TINYINT);
EXPECT_EQ(pSchema->colId, 3);
EXPECT_EQ(pSchema->bytes, 2);
EXPECT_STREQ(pSchema->name, "tag1");
}
{
SSchema* pSchema = &pRsp->pSchema[3];
EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BIGINT);
EXPECT_EQ(pSchema->colId, 4);
EXPECT_EQ(pSchema->bytes, 8);
EXPECT_STREQ(pSchema->name, "tag2");
}
{
SSchema* pSchema = &pRsp->pSchema[4];
EXPECT_EQ(pSchema->type, TSDB_DATA_TYPE_BINARY);
EXPECT_EQ(pSchema->colId, 5);
EXPECT_EQ(pSchema->bytes, 16);
EXPECT_STREQ(pSchema->name, "tag3");
}
}
// restart

12
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -209,7 +209,7 @@ typedef struct {
} SDbCfg;
typedef struct {
char name[TSDB_FULL_DB_NAME_LEN];
char name[TSDB_DB_FNAME_LEN];
char acct[TSDB_USER_LEN];
int64_t createdTime;
int64_t updateTime;
@ -232,7 +232,7 @@ typedef struct {
int32_t version;
uint32_t hashBegin;
uint32_t hashEnd;
char dbName[TSDB_FULL_DB_NAME_LEN];
char dbName[TSDB_DB_FNAME_LEN];
int64_t dbUid;
int32_t numOfTables;
int32_t numOfTimeSeries;
@ -246,7 +246,7 @@ typedef struct {
typedef struct {
char name[TSDB_TABLE_FNAME_LEN];
char db[TSDB_FULL_DB_NAME_LEN];
char db[TSDB_DB_FNAME_LEN];
int64_t createdTime;
int64_t updateTime;
uint64_t uid;
@ -286,7 +286,7 @@ typedef struct {
int32_t payloadLen;
void *pIter;
SMnode *pMnode;
char db[TSDB_FULL_DB_NAME_LEN];
char db[TSDB_DB_FNAME_LEN];
int16_t offset[TSDB_MAX_COLUMNS];
int32_t bytes[TSDB_MAX_COLUMNS];
char payload[];
@ -294,7 +294,7 @@ typedef struct {
typedef struct {
char name[TSDB_TOPIC_FNAME_LEN];
char db[TSDB_FULL_DB_NAME_LEN];
char db[TSDB_DB_FNAME_LEN];
int64_t createTime;
int64_t updateTime;
uint64_t uid;
@ -309,7 +309,7 @@ typedef struct {
typedef struct SMnodeMsg {
char user[TSDB_USER_LEN];
char db[TSDB_FULL_DB_NAME_LEN];
char db[TSDB_DB_FNAME_LEN];
int32_t acctId;
SMnode *pMnode;
int64_t createdTime;

10
source/dnode/mnode/impl/src/mndDb.c
View File

@ -69,7 +69,7 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
if (pRaw == NULL) return NULL;
int32_t dataPos = 0;
SDB_SET_BINARY(pRaw, dataPos, pDb->name, TSDB_FULL_DB_NAME_LEN)
SDB_SET_BINARY(pRaw, dataPos, pDb->name, TSDB_DB_FNAME_LEN)
SDB_SET_BINARY(pRaw, dataPos, pDb->acct, TSDB_USER_LEN)
SDB_SET_INT64(pRaw, dataPos, pDb->createdTime)
SDB_SET_INT64(pRaw, dataPos, pDb->updateTime)
@ -116,7 +116,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
if (pDb == NULL) return NULL;
int32_t dataPos = 0;
SDB_GET_BINARY(pRaw, pRow, dataPos, pDb->name, TSDB_FULL_DB_NAME_LEN)
SDB_GET_BINARY(pRaw, pRow, dataPos, pDb->name, TSDB_DB_FNAME_LEN)
SDB_GET_BINARY(pRaw, pRow, dataPos, pDb->acct, TSDB_USER_LEN)
SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->createdTime)
SDB_GET_INT64(pRaw, pRow, dataPos, &pDb->updateTime)
@ -353,11 +353,11 @@ static int32_t mndSetCreateDbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj
static int32_t mndCreateDb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDbMsg *pCreate, SUserObj *pUser) {
SDbObj dbObj = {0};
memcpy(dbObj.name, pCreate->db, TSDB_FULL_DB_NAME_LEN);
memcpy(dbObj.name, pCreate->db, TSDB_DB_FNAME_LEN);
memcpy(dbObj.acct, pUser->acct, TSDB_USER_LEN);
dbObj.createdTime = taosGetTimestampMs();
dbObj.updateTime = dbObj.createdTime;
dbObj.uid = mndGenerateUid(dbObj.name, TSDB_FULL_DB_NAME_LEN);
dbObj.uid = mndGenerateUid(dbObj.name, TSDB_DB_FNAME_LEN);
dbObj.cfgVersion = 1;
dbObj.vgVersion = 1;
dbObj.hashMethod = 1;
@ -891,7 +891,7 @@ static int32_t mndProcessUseDbMsg(SMnodeMsg *pMsg) {
}
}
memcpy(pRsp->db, pDb->name, TSDB_FULL_DB_NAME_LEN);
memcpy(pRsp->db, pDb->name, TSDB_DB_FNAME_LEN);
pRsp->vgVersion = htonl(pDb->vgVersion);
pRsp->vgNum = htonl(vindex);
pRsp->hashMethod = pDb->hashMethod;

29
source/dnode/mnode/impl/src/mndDnode.c
View File

@ -386,20 +386,16 @@ static int32_t mndCreateDnode(SMnode *pMnode, SMnodeMsg *pMsg, SCreateDnodeMsg *
dnodeObj.id = sdbGetMaxId(pMnode->pSdb, SDB_DNODE);
dnodeObj.createdTime = taosGetTimestampMs();
dnodeObj.updateTime = dnodeObj.createdTime;
taosGetFqdnPortFromEp(pCreate->ep, dnodeObj.fqdn, &dnodeObj.port);
if (dnodeObj.fqdn[0] == 0 || dnodeObj.port <= 0) {
terrno = TSDB_CODE_MND_INVALID_DNODE_EP;
mError("dnode:%s, failed to create since %s", pCreate->ep, terrstr());
return terrno;
}
dnodeObj.port = pCreate->port;
memcpy(dnodeObj.fqdn, pCreate->fqdn, TSDB_FQDN_LEN);
snprintf(dnodeObj.ep, TSDB_EP_LEN, "%s:%u", dnodeObj.fqdn, dnodeObj.port);
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg);
if (pTrans == NULL) {
mError("dnode:%s, failed to create since %s", pCreate->ep, terrstr());
mError("dnode:%s, failed to create since %s", dnodeObj.ep, terrstr());
return -1;
}
mDebug("trans:%d, used to create dnode:%s", pTrans->id, pCreate->ep);
mDebug("trans:%d, used to create dnode:%s", pTrans->id, dnodeObj.ep);
SSdbRaw *pRedoRaw = mndDnodeActionEncode(&dnodeObj);
if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) {
@ -423,17 +419,20 @@ static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;
SCreateDnodeMsg *pCreate = pMsg->rpcMsg.pCont;
mDebug("dnode:%s, start to create", pCreate->ep);
mDebug("dnode:%s:%d, start to create", pCreate->fqdn, pCreate->port);
if (pCreate->ep[0] == 0) {
pCreate->port = htonl(pCreate->port);
if (pCreate->fqdn[0] == 0 || pCreate->port <= 0 || pCreate->port > UINT16_MAX) {
terrno = TSDB_CODE_MND_INVALID_DNODE_EP;
mError("dnode:%s, failed to create since %s", pCreate->ep, terrstr());
mError("dnode:%s:%d, failed to create since %s", pCreate->fqdn, pCreate->port, terrstr());
return -1;
}
SDnodeObj *pDnode = mndAcquireDnodeByEp(pMnode, pCreate->ep);
char ep[TSDB_EP_LEN];
snprintf(ep, TSDB_EP_LEN, "%s:%d", pCreate->fqdn, pCreate->port);
SDnodeObj *pDnode = mndAcquireDnodeByEp(pMnode, ep);
if (pDnode != NULL) {
mError("dnode:%d, already exist", pDnode->id);
mError("dnode:%d, already exist, %s:%u", pDnode->id, pCreate->fqdn, pCreate->port);
mndReleaseDnode(pMnode, pDnode);
terrno = TSDB_CODE_MND_DNODE_ALREADY_EXIST;
return -1;
@ -442,7 +441,7 @@ static int32_t mndProcessCreateDnodeMsg(SMnodeMsg *pMsg) {
int32_t code = mndCreateDnode(pMnode, pMsg, pCreate);
if (code != 0) {
mError("dnode:%s, failed to create since %s", pCreate->ep, terrstr());
mError("dnode:%s:%d, failed to create since %s", pCreate->fqdn, pCreate->port, terrstr());
return -1;
}

2
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -194,7 +194,7 @@ static int32_t mndProcessConnectMsg(SMnodeMsg *pMsg) {
taosIp2String(info.clientIp, ip);
if (pReq->db[0]) {
snprintf(pMsg->db, TSDB_FULL_DB_NAME_LEN, "%d%s%s", pMsg->acctId, TS_PATH_DELIMITER, pReq->db);
snprintf(pMsg->db, TSDB_DB_FNAME_LEN, "%d%s%s", pMsg->acctId, TS_PATH_DELIMITER, pReq->db);
SDbObj *pDb = mndAcquireDb(pMnode, pMsg->db);
if (pDb == NULL) {
terrno = TSDB_CODE_MND_INVALID_DB;

2
source/dnode/mnode/impl/src/mndShow.c
View File

@ -62,7 +62,7 @@ static SShowObj *mndCreateShowObj(SMnode *pMnode, SShowMsg *pMsg) {
pShow->pMnode = pMnode;
pShow->type = pMsg->type;
pShow->payloadLen = pMsg->payloadLen;
memcpy(pShow->db, pMsg->db, TSDB_FULL_DB_NAME_LEN);
memcpy(pShow->db, pMsg->db, TSDB_DB_FNAME_LEN);
memcpy(pShow->payload, pMsg->payload, pMsg->payloadLen);
} else {
terrno = TSDB_CODE_OUT_OF_MEMORY;

17
source/dnode/mnode/impl/src/mndStb.c
View File

@ -76,7 +76,7 @@ static SSdbRaw *mndStbActionEncode(SStbObj *pStb) {
int32_t dataPos = 0;
SDB_SET_BINARY(pRaw, dataPos, pStb->name, TSDB_TABLE_FNAME_LEN)
SDB_SET_BINARY(pRaw, dataPos, pStb->db, TSDB_FULL_DB_NAME_LEN)
SDB_SET_BINARY(pRaw, dataPos, pStb->db, TSDB_DB_FNAME_LEN)
SDB_SET_INT64(pRaw, dataPos, pStb->createdTime)
SDB_SET_INT64(pRaw, dataPos, pStb->updateTime)
SDB_SET_INT64(pRaw, dataPos, pStb->uid)
@ -117,7 +117,7 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) {
int32_t dataPos = 0;
SDB_GET_BINARY(pRaw, pRow, dataPos, pStb->name, TSDB_TABLE_FNAME_LEN)
SDB_GET_BINARY(pRaw, pRow, dataPos, pStb->db, TSDB_FULL_DB_NAME_LEN)
SDB_GET_BINARY(pRaw, pRow, dataPos, pStb->db, TSDB_DB_FNAME_LEN)
SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->createdTime)
SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->updateTime)
SDB_GET_INT64(pRaw, pRow, dataPos, &pStb->uid)
@ -291,7 +291,6 @@ static int32_t mndCheckCreateStbMsg(SCreateStbMsg *pCreate) {
int32_t totalCols = pCreate->numOfColumns + pCreate->numOfTags;
for (int32_t i = 0; i < totalCols; ++i) {
SSchema *pSchema = &pCreate->pSchema[i];
pSchema->colId = htonl(pSchema->colId);
pSchema->bytes = htonl(pSchema->bytes);
}
@ -317,10 +316,6 @@ static int32_t mndCheckCreateStbMsg(SCreateStbMsg *pCreate) {
terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
return -1;
}
if (pSchema->colId < 0 || pSchema->colId >= maxColId) {
terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
return -1;
}
if (pSchema->bytes <= 0) {
terrno = TSDB_CODE_MND_INVALID_STB_OPTION;
return -1;
@ -435,7 +430,7 @@ static int32_t mndSetCreateStbUndoActions(SMnode *pMnode, STrans *pTrans, SDbObj
static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateStbMsg *pCreate, SDbObj *pDb) {
SStbObj stbObj = {0};
tstrncpy(stbObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN);
tstrncpy(stbObj.db, pDb->name, TSDB_FULL_DB_NAME_LEN);
tstrncpy(stbObj.db, pDb->name, TSDB_DB_FNAME_LEN);
stbObj.createdTime = taosGetTimestampMs();
stbObj.updateTime = stbObj.createdTime;
stbObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);
@ -453,6 +448,10 @@ static int32_t mndCreateStb(SMnode *pMnode, SMnodeMsg *pMsg, SCreateStbMsg *pCre
}
memcpy(stbObj.pSchema, pCreate->pSchema, totalSize);
for (int32_t i = 0; i < totalCols; ++i) {
stbObj.pSchema[i].colId = i + 1;
}
int32_t code = 0;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, &pMsg->rpcMsg);
if (pTrans == NULL) {
@ -767,7 +766,7 @@ static int32_t mndProcessStbMetaMsg(SMnodeMsg *pMsg) {
return -1;
}
memcpy(pMeta->stbFname, pStb->name, TSDB_TABLE_FNAME_LEN);
memcpy(pMeta->tbFname, pStb->name, TSDB_TABLE_FNAME_LEN);
pMeta->numOfTags = htonl(pStb->numOfTags);
pMeta->numOfColumns = htonl(pStb->numOfColumns);
pMeta->precision = pDb->cfg.precision;

6
source/dnode/mnode/impl/src/mndTopic.c
View File

@ -79,7 +79,7 @@ static SSdbRaw *mndTopicActionEncode(STopicObj *pTopic) {
int32_t dataPos = 0;
SDB_SET_BINARY(pRaw, dataPos, pTopic->name, TSDB_TABLE_FNAME_LEN);
SDB_SET_BINARY(pRaw, dataPos, pTopic->db, TSDB_FULL_DB_NAME_LEN);
SDB_SET_BINARY(pRaw, dataPos, pTopic->db, TSDB_DB_FNAME_LEN);
SDB_SET_INT64(pRaw, dataPos, pTopic->createTime);
SDB_SET_INT64(pRaw, dataPos, pTopic->updateTime);
SDB_SET_INT64(pRaw, dataPos, pTopic->uid);
@ -113,7 +113,7 @@ static SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) {
int32_t dataPos = 0;
SDB_GET_BINARY(pRaw, pRow, dataPos, pTopic->name, TSDB_TABLE_FNAME_LEN);
SDB_GET_BINARY(pRaw, pRow, dataPos, pTopic->db, TSDB_FULL_DB_NAME_LEN);
SDB_GET_BINARY(pRaw, pRow, dataPos, pTopic->db, TSDB_DB_FNAME_LEN);
SDB_GET_INT64(pRaw, pRow, dataPos, &pTopic->createTime);
SDB_GET_INT64(pRaw, pRow, dataPos, &pTopic->updateTime);
SDB_GET_INT64(pRaw, pRow, dataPos, &pTopic->uid);
@ -348,7 +348,7 @@ static int32_t mndSetCreateTopicUndoActions(SMnode *pMnode, STrans *pTrans, SDbO
static int32_t mndCreateTopic(SMnode *pMnode, SMnodeMsg *pMsg, SCreateTopicMsg *pCreate, SDbObj *pDb) {
STopicObj topicObj = {0};
tstrncpy(topicObj.name, pCreate->name, TSDB_TABLE_FNAME_LEN);
tstrncpy(topicObj.db, pDb->name, TSDB_FULL_DB_NAME_LEN);
tstrncpy(topicObj.db, pDb->name, TSDB_DB_FNAME_LEN);
topicObj.createTime = taosGetTimestampMs();
topicObj.updateTime = topicObj.createTime;
topicObj.uid = mndGenerateUid(pCreate->name, TSDB_TABLE_FNAME_LEN);

10
source/dnode/mnode/impl/src/mndVgroup.c
View File

@ -80,7 +80,7 @@ SSdbRaw *mndVgroupActionEncode(SVgObj *pVgroup) {
SDB_SET_INT32(pRaw, dataPos, pVgroup->version)
SDB_SET_INT32(pRaw, dataPos, pVgroup->hashBegin)
SDB_SET_INT32(pRaw, dataPos, pVgroup->hashEnd)
SDB_SET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_FULL_DB_NAME_LEN)
SDB_SET_BINARY(pRaw, dataPos, pVgroup->dbName, TSDB_DB_FNAME_LEN)
SDB_SET_INT64(pRaw, dataPos, pVgroup->dbUid)
SDB_SET_INT8(pRaw, dataPos, pVgroup->replica)
for (int8_t i = 0; i < pVgroup->replica; ++i) {
@ -115,7 +115,7 @@ SSdbRow *mndVgroupActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT32(pRaw, pRow, dataPos, &pVgroup->version)
SDB_GET_INT32(pRaw, pRow, dataPos, &pVgroup->hashBegin)
SDB_GET_INT32(pRaw, pRow, dataPos, &pVgroup->hashEnd)
SDB_GET_BINARY(pRaw, pRow, dataPos, pVgroup->dbName, TSDB_FULL_DB_NAME_LEN)
SDB_GET_BINARY(pRaw, pRow, dataPos, pVgroup->dbName, TSDB_DB_FNAME_LEN)
SDB_GET_INT64(pRaw, pRow, dataPos, &pVgroup->dbUid)
SDB_GET_INT8(pRaw, pRow, dataPos, &pVgroup->replica)
for (int8_t i = 0; i < pVgroup->replica; ++i) {
@ -172,7 +172,7 @@ SCreateVnodeMsg *mndBuildCreateVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbOb
pCreate->vgId = htonl(pVgroup->vgId);
pCreate->dnodeId = htonl(pDnode->id);
memcpy(pCreate->db, pDb->name, TSDB_FULL_DB_NAME_LEN);
memcpy(pCreate->db, pDb->name, TSDB_DB_FNAME_LEN);
pCreate->dbUid = htobe64(pDb->uid);
pCreate->vgVersion = htonl(pVgroup->version);
pCreate->cacheBlockSize = htonl(pDb->cfg.cacheBlockSize);
@ -231,7 +231,7 @@ SDropVnodeMsg *mndBuildDropVnodeMsg(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *p
pDrop->dnodeId = htonl(pDnode->id);
pDrop->vgId = htonl(pVgroup->vgId);
memcpy(pDrop->db, pDb->name, TSDB_FULL_DB_NAME_LEN);
memcpy(pDrop->db, pDb->name, TSDB_DB_FNAME_LEN);
pDrop->dbUid = htobe64(pDb->uid);
return pDrop;
@ -294,7 +294,7 @@ int32_t mndAllocVgroup(SMnode *pMnode, SDbObj *pDb, SVgObj **ppVgroups) {
pVgroup->hashEnd = hashMin + hashInterval * (v + 1) - 1;
}
memcpy(pVgroup->dbName, pDb->name, TSDB_FULL_DB_NAME_LEN);
memcpy(pVgroup->dbName, pDb->name, TSDB_DB_FNAME_LEN);
pVgroup->dbUid = pDb->uid;
pVgroup->replica = pDb->cfg.replications;

8
source/dnode/mnode/impl/src/mnode.c
View File

@ -390,7 +390,7 @@ void mndSendRsp(SMnodeMsg *pMsg, int32_t code) {
rpcSendResponse(&rpcRsp);
}
static void mndProcessRpcMsg(SMnodeMsg *pMsg) {
void mndProcessMsg(SMnodeMsg *pMsg) {
SMnode *pMnode = pMsg->pMnode;
int32_t code = 0;
tmsg_t msgType = pMsg->rpcMsg.msgType;
@ -451,12 +451,6 @@ void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) {
}
}
void mndProcessReadMsg(SMnodeMsg *pMsg) { mndProcessRpcMsg(pMsg); }
void mndProcessWriteMsg(SMnodeMsg *pMsg) { mndProcessRpcMsg(pMsg); }
void mndProcessSyncMsg(SMnodeMsg *pMsg) { mndProcessRpcMsg(pMsg); }
uint64_t mndGenerateUid(char *name, int32_t len) {
int64_t us = taosGetTimestampUs();
int32_t hashval = MurmurHash3_32(name, len);

4
source/dnode/snode/src/snode.c
View File

@ -15,7 +15,7 @@
#include "sndInt.h"
SSnode *sndOpen(const SSnodeOpt *pOption) {
SSnode *sndOpen(const char *path, const SSnodeOpt *pOption) {
SSnode *pSnode = calloc(1, sizeof(SSnode));
return pSnode;
}
@ -28,3 +28,5 @@ int32_t sndProcessWriteMsg(SSnode *pSnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
*pRsp = NULL;
return 0;
}
void sndDestroy(const char *path) {}

2
source/dnode/vnode/impl/inc/vnodeDef.h
View File

@ -73,7 +73,7 @@ struct SVnode {
SVnodeSync* pSync;
SVnodeFS* pFs;
tsem_t canCommit;
void* pQuery;
SQHandle* pQuery;
};
int vnodeScheduleTask(SVnodeTask* task);

3
source/dnode/vnode/impl/inc/vnodeQuery.h
View File

@ -22,6 +22,9 @@ extern "C" {
#include "vnodeInt.h"
#include "qworker.h"
typedef struct SQWorkerMgmt SQHandle;
int vnodeQueryOpen(SVnode *pVnode);
#ifdef __cplusplus

20
source/dnode/vnode/impl/src/vnodeQuery.c
View File

@ -20,13 +20,27 @@ int vnodeQueryOpen(SVnode *pVnode) { return qWorkerInit(NULL, &pVnode->pQuery);
int vnodeProcessQueryReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
vInfo("query message is processed");
qWorkerProcessQueryMsg(pVnode, pVnode->pQuery, pMsg);
return 0;
return qWorkerProcessQueryMsg(pVnode, pVnode->pQuery, pMsg);
}
int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
vInfo("fetch message is processed");
qWorkerProcessFetchMsg(pVnode, pVnode->pQuery, pMsg);
switch (pMsg->msgType) {
case TDMT_VND_FETCH:
return qWorkerProcessFetchMsg(pVnode, pVnode->pQuery, pMsg);
case TDMT_VND_RES_READY:
return qWorkerProcessReadyMsg(pVnode, pVnode->pQuery, pMsg);
case TDMT_VND_TASKS_STATUS:
return qWorkerProcessStatusMsg(pVnode, pVnode->pQuery, pMsg);
case TDMT_VND_CANCEL_TASK:
return qWorkerProcessCancelMsg(pVnode, pVnode->pQuery, pMsg);
case TDMT_VND_DROP_TASK:
return qWorkerProcessDropMsg(pVnode, pVnode->pQuery, pMsg);
default:
vError("unknown msg type:%d in fetch queue", pMsg->msgType);
return TSDB_CODE_VND_APP_ERROR;
break;
}
return 0;
}

2
source/libs/catalog/inc/catalogInt.h
View File

@ -46,7 +46,6 @@ typedef struct STableMetaCache {
} STableMetaCache;
typedef struct SCatalog {
SVgroupListCache vgroupCache;
SDBVgroupCache dbCache;
STableMetaCache tableCache;
} SCatalog;
@ -67,6 +66,7 @@ typedef uint32_t (*tableNameHashFp)(const char *, uint32_t);
#define ctgTrace(...) do { if (ctgDebugFlag & DEBUG_TRACE) { taosPrintLog("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0)
#define ctgDebugL(...) do { if (ctgDebugFlag & DEBUG_DEBUG) { taosPrintLongString("CTG ", ctgDebugFlag, __VA_ARGS__); }} while(0)
#define CTG_CACHE_ENABLED() (ctgMgmt.cfg.maxDBCacheNum > 0 || ctgMgmt.cfg.maxTblCacheNum > 0)
#define CTG_ERR_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; return _code; } } while (0)
#define CTG_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0)

117
source/libs/catalog/src/catalog.c
View File

@ -49,22 +49,11 @@ int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEp
SEpSet *pVnodeEpSet = NULL;
int32_t msgLen = 0;
CTG_ERR_RET(queryBuildMsg[TDMT_MND_USE_DB](input, &msg, 0, &msgLen));
char *pMsg = rpcMallocCont(msgLen);
if (NULL == pMsg) {
ctgError("rpc malloc %d failed", msgLen);
tfree(msg);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
memcpy(pMsg, msg, msgLen);
tfree(msg);
CTG_ERR_RET(queryBuildMsg[TMSG_INDEX(TDMT_MND_USE_DB)](input, &msg, 0, &msgLen));
SRpcMsg rpcMsg = {
.msgType = TDMT_MND_USE_DB,
.pCont = pMsg,
.pCont = msg,
.contLen = msgLen,
};
@ -76,7 +65,7 @@ int32_t ctgGetDBVgroupFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEp
CTG_ERR_RET(rpcRsp.code);
}
CTG_ERR_RET(queryProcessMsgRsp[TDMT_MND_USE_DB](out, rpcRsp.pCont, rpcRsp.contLen));
CTG_ERR_RET(queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_USE_DB)](out, rpcRsp.pCont, rpcRsp.contLen));
return TSDB_CODE_SUCCESS;
}
@ -146,8 +135,44 @@ void ctgGenEpSet(SEpSet *epSet, SVgroupInfo *vgroupInfo) {
}
}
int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char *pDBName, const char* pTableName, STableMetaOutput* output) {
if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDBName || NULL == pTableName || NULL == output) {
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char *pDBName, const char* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) {
char tbFullName[TSDB_TABLE_FNAME_LEN];
snprintf(tbFullName, sizeof(tbFullName), "%s.%s", pDBName, pTableName);
SBuildTableMetaInput bInput = {.vgId = 0, .tableFullName = tbFullName};
char *msg = NULL;
SEpSet *pVnodeEpSet = NULL;
int32_t msgLen = 0;
CTG_ERR_RET(queryBuildMsg[TMSG_INDEX(TDMT_MND_STB_META)](&bInput, &msg, 0, &msgLen));
SRpcMsg rpcMsg = {
.msgType = TDMT_MND_STB_META,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pRpc, (SEpSet*)pMgmtEps, &rpcMsg, &rpcRsp);
if (TSDB_CODE_SUCCESS != rpcRsp.code) {
ctgError("error rsp for table meta, code:%x", rpcRsp.code);
CTG_ERR_RET(rpcRsp.code);
}
CTG_ERR_RET(queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_STB_META)](output, rpcRsp.pCont, rpcRsp.contLen));
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetTableMetaFromVnode(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char *pDBName, const char* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* output) {
if (NULL == pCatalog || NULL == pRpc || NULL == pMgmtEps || NULL == pDBName || NULL == pTableName || NULL == vgroupInfo || NULL == output) {
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
}
@ -161,7 +186,7 @@ int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SE
SEpSet *pVnodeEpSet = NULL;
int32_t msgLen = 0;
CTG_ERR_RET(queryBuildMsg[TDMT_VND_TABLE_META](&bInput, &msg, 0, &msgLen));
CTG_ERR_RET(queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)](&bInput, &msg, 0, &msgLen));
SRpcMsg rpcMsg = {
.msgType = TDMT_VND_TABLE_META,
@ -181,7 +206,7 @@ int32_t ctgGetTableMetaFromMnode(struct SCatalog* pCatalog, void *pRpc, const SE
CTG_ERR_RET(rpcRsp.code);
}
CTG_ERR_RET(queryProcessMsgRsp[TDMT_VND_TABLE_META](output, rpcRsp.pCont, rpcRsp.contLen));
CTG_ERR_RET(queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)](output, rpcRsp.pCont, rpcRsp.contLen));
return TSDB_CODE_SUCCESS;
}
@ -307,7 +332,9 @@ int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *out
ctgError("init hash[%d] for tablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
}
if (NULL == pCatalog->tableCache.stableCache) {
pCatalog->tableCache.stableCache = taosHashInit(ctgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK);
if (NULL == pCatalog->tableCache.stableCache) {
ctgError("init hash[%d] for stablemeta cache failed", ctgMgmt.cfg.maxTblCacheNum);
@ -318,55 +345,51 @@ int32_t ctgUpdateTableMetaCache(struct SCatalog *pCatalog, STableMetaOutput *out
if (output->metaNum == 2) {
if (taosHashPut(pCatalog->tableCache.cache, output->ctbFname, strlen(output->ctbFname), &output->ctbMeta, sizeof(output->ctbMeta)) != 0) {
ctgError("push ctable[%s] to table cache failed", output->ctbFname);
goto error_exit;
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
if (TSDB_SUPER_TABLE != output->tbMeta->tableType) {
ctgError("table type[%d] error, expected:%d", output->tbMeta->tableType, TSDB_SUPER_TABLE);
goto error_exit;
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
}
int32_t tbSize = sizeof(*output->tbMeta) + sizeof(SSchema) * (output->tbMeta->tableInfo.numOfColumns + output->tbMeta->tableInfo.numOfTags);
if (taosHashPut(pCatalog->tableCache.cache, output->tbFname, strlen(output->tbFname), output->tbMeta, tbSize) != 0) {
ctgError("push table[%s] to table cache failed", output->tbFname);
goto error_exit;
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
if (TSDB_SUPER_TABLE == output->tbMeta->tableType) {
if (taosHashPut(pCatalog->tableCache.stableCache, &output->tbMeta->suid, sizeof(output->tbMeta->suid), &output->tbMeta, POINTER_BYTES) != 0) {
ctgError("push suid[%"PRIu64"] to stable cache failed", output->tbMeta->suid);
goto error_exit;
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
}
return TSDB_CODE_SUCCESS;
error_exit:
if (pCatalog->vgroupCache.cache) {
taosHashCleanup(pCatalog->vgroupCache.cache);
pCatalog->vgroupCache.cache = NULL;
}
pCatalog->vgroupCache.vgroupVersion = CTG_DEFAULT_INVALID_VERSION;
CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR);
}
int32_t catalogInit(SCatalogCfg *cfg) {
ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (NULL == ctgMgmt.pCluster) {
CTG_ERR_LRET(TSDB_CODE_CTG_INTERNAL_ERROR, "init %d cluster cache failed", CTG_DEFAULT_CACHE_CLUSTER_NUMBER);
if (ctgMgmt.pCluster) {
ctgError("catalog already init");
CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT);
}
if (cfg) {
memcpy(&ctgMgmt.cfg, cfg, sizeof(*cfg));
} else {
ctgMgmt.cfg.enableVgroupCache = true;
ctgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER;
ctgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TABLEMETA_NUMBER;
}
if (CTG_CACHE_ENABLED()) {
ctgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (NULL == ctgMgmt.pCluster) {
CTG_ERR_LRET(TSDB_CODE_CTG_INTERNAL_ERROR, "init %d cluster cache failed", CTG_DEFAULT_CACHE_CLUSTER_NUMBER);
}
}
return TSDB_CODE_SUCCESS;
}
@ -381,21 +404,19 @@ int32_t catalogGetHandle(const char* clusterId , struct SCatalog** catalogHandle
}
size_t clen = strlen(clusterId);
SCatalog *clusterCtg = (SCatalog *)taosHashGet(ctgMgmt.pCluster, clusterId, clen);
SCatalog **ctg = (SCatalog **)taosHashGet(ctgMgmt.pCluster, clusterId, clen);
if (clusterCtg) {
*catalogHandle = clusterCtg;
if (ctg && (*ctg)) {
*catalogHandle = *ctg;
return TSDB_CODE_SUCCESS;
}
clusterCtg = calloc(1, sizeof(*clusterCtg));
SCatalog *clusterCtg = calloc(1, sizeof(SCatalog));
if (NULL == clusterCtg) {
ctgError("calloc %d failed", (int32_t)sizeof(*clusterCtg));
ctgError("calloc %d failed", (int32_t)sizeof(SCatalog));
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
}
clusterCtg->vgroupCache.vgroupVersion = CTG_DEFAULT_INVALID_VERSION;
if (taosHashPut(ctgMgmt.pCluster, clusterId, clen, &clusterCtg, POINTER_BYTES)) {
ctgError("put cluster %s cache to hash failed", clusterId);
tfree(clusterCtg);
@ -443,7 +464,7 @@ int32_t catalogUpdateDBVgroupCache(struct SCatalog* pCatalog, const char* dbName
}
if (NULL == pCatalog->dbCache.cache) {
pCatalog->dbCache.cache = taosHashInit(CTG_DEFAULT_CACHE_DB_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
pCatalog->dbCache.cache = taosHashInit(ctgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK);
if (NULL == pCatalog->dbCache.cache) {
ctgError("init hash[%d] for db cache failed", CTG_DEFAULT_CACHE_DB_NUMBER);
CTG_ERR_RET(TSDB_CODE_CTG_MEM_ERROR);
@ -500,8 +521,8 @@ int32_t catalogGetDBVgroup(struct SCatalog* pCatalog, void *pRpc, const SEpSet*
return TSDB_CODE_SUCCESS;
}
int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) {
return ctgGetTableMetaImpl(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, false, pTableMeta);
int32_t catalogGetTableMeta(struct SCatalog* pCatalog, void *pTransporter, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName, STableMeta** pTableMeta) {
return ctgGetTableMetaImpl(pCatalog, pTransporter, pMgmtEps, pDBName, pTableName, false, pTableMeta);
}
int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* pMgmtEps, const char* pDBName, const char* pTableName) {
@ -515,7 +536,9 @@ int32_t catalogRenewTableMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSe
STableMetaOutput output = {0};
CTG_ERR_RET(ctgGetTableMetaFromMnode(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &vgroupInfo, &output));
//CTG_ERR_RET(ctgGetTableMetaFromVnode(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &vgroupInfo, &output));
CTG_ERR_RET(ctgGetTableMetaFromMnode(pCatalog, pRpc, pMgmtEps, pDBName, pTableName, &output));
CTG_ERR_RET(ctgUpdateTableMetaCache(pCatalog, &output));
@ -602,7 +625,7 @@ int32_t catalogGetAllMeta(struct SCatalog* pCatalog, void *pRpc, const SEpSet* p
int32_t code = 0;
if (pReq->pTableName) {
char dbName[TSDB_FULL_DB_NAME_LEN];
char dbName[TSDB_DB_FNAME_LEN];
int32_t tbNum = (int32_t)taosArrayGetSize(pReq->pTableName);
if (tbNum > 0) {
pRsp->pTableMeta = taosArrayInit(tbNum, POINTER_BYTES);

3
source/libs/index/inc/index_cache.h
View File

@ -39,6 +39,7 @@ typedef struct IndexCache {
int32_t nTerm;
int8_t type;
pthread_mutex_t mtx;
} IndexCache;
#define CACHE_VERSION(cache) atomic_load_32(&cache->version)
@ -71,7 +72,7 @@ void indexCacheUnRef(IndexCache* cache);
void indexCacheDebug(IndexCache* cache);
void indexCacheDestroySkiplist(SSkipList* slt);
void indexCacheDestroyImm(IndexCache* cache);
#ifdef __cplusplus
}
#endif

4
source/libs/index/src/index.c
View File

@ -436,9 +436,7 @@ int indexFlushCacheTFile(SIndex* sIdx, void* cache) {
if (ret != 0) { indexError("faile to write into tindex "); }
}
// not free later, just put int table cache
SSkipList* timm = (SSkipList*)pCache->imm;
pCache->imm = NULL; // or throw int bg thread
indexCacheDestroySkiplist(timm);
indexCacheDestroyImm(pCache);
tfileWriteClose(tw);
indexCacheIteratorDestroy(cacheIter);

51
source/libs/index/src/index_cache.c
View File

@ -20,7 +20,7 @@
#define MAX_INDEX_KEY_LEN 256 // test only, change later
#define CACH_LIMIT 1000000
#define MEM_TERM_LIMIT 1000000
// ref index_cache.h:22
//#define CACHE_KEY_LEN(p) \
// (sizeof(int32_t) + sizeof(uint16_t) + sizeof(p->colType) + sizeof(p->nColVal) + p->nColVal + sizeof(uint64_t) + sizeof(p->operType))
@ -78,6 +78,7 @@ IndexCache* indexCacheCreate(SIndex* idx, const char* colName, int8_t type) {
cache->index = idx;
cache->version = 0;
pthread_mutex_init(&cache->mtx, NULL);
indexCacheRef(cache);
return cache;
}
@ -103,13 +104,21 @@ void indexCacheDestroySkiplist(SSkipList* slt) {
}
tSkipListDestroyIter(iter);
}
void indexCacheDestroyImm(IndexCache* cache) {
pthread_mutex_lock(&cache->mtx);
SSkipList* timm = (SSkipList*)cache->imm;
cache->imm = NULL; // or throw int bg thread
pthread_mutex_unlock(&cache->mtx);
indexCacheDestroySkiplist(timm);
}
void indexCacheDestroy(void* cache) {
IndexCache* pCache = cache;
if (pCache == NULL) { return; }
tSkipListDestroy(pCache->mem);
tSkipListDestroy(pCache->imm);
free(pCache->colName);
free(pCache);
}
@ -170,6 +179,27 @@ int indexCacheSchedToMerge(IndexCache* pCache) {
taosScheduleTask(indexQhandle, &schedMsg);
}
static void indexCacheMakeRoomForWrite(IndexCache* cache) {
while (true) {
if (cache->nTerm < MEM_TERM_LIMIT) {
cache->nTerm += 1;
break;
} else if (cache->imm != NULL) {
// TODO: wake up by condition variable
pthread_mutex_unlock(&cache->mtx);
taosMsleep(50);
pthread_mutex_lock(&cache->mtx);
} else {
cache->imm = cache->mem;
cache->mem = indexInternalCacheCreate(cache->type);
cache->nTerm = 1;
// sched to merge
// unref cache in bgwork
indexCacheSchedToMerge(cache);
}
}
}
int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) {
if (cache == NULL) { return -1; }
@ -188,23 +218,12 @@ int indexCachePut(void* cache, SIndexTerm* term, uint64_t uid) {
ct->uid = uid;
ct->operaType = term->operType;
// ugly code, refactor later
pthread_mutex_lock(&pCache->mtx);
indexCacheMakeRoomForWrite(pCache);
tSkipListPut(pCache->mem, (char*)ct);
pCache->nTerm += 1;
pthread_mutex_unlock(&pCache->mtx);
if (pCache->nTerm >= CACH_LIMIT) {
pCache->nTerm = 0;
while (pCache->imm != NULL) {
// do nothong
}
pCache->imm = pCache->mem;
pCache->mem = indexInternalCacheCreate(pCache->type);
// sched to merge
// unref cache int bgwork
indexCacheSchedToMerge(pCache);
}
indexCacheUnRef(pCache);
return 0;
// encode end

14
source/libs/parser/inc/sql.y
View File

@ -6,7 +6,7 @@
%default_type {SToken}
%extra_argument {SSqlInfo* pInfo}
%fallback ID BOOL TINYINT SMALLINT INTEGER BIGINT FLOAT DOUBLE STRING TIMESTAMP BINARY NCHAR.
%fallback ID BOOL INTEGER FLOAT STRING TIMESTAMP.
%left OR.
%left AND.
@ -200,7 +200,8 @@ ifnotexists(X) ::= . { X.n = 0;}
/////////////////////////////////THE CREATE STATEMENT///////////////////////////////////////
//create option for dnode/db/user/account
cmd ::= CREATE DNODE ids(X) PORT ids(Y). { setDCLSqlElems(pInfo, TSDB_SQL_CREATE_DNODE, 2, &X, &Y);}
cmd ::= CREATE DNODE ids(X) PORT ids(Y). { setDCLSqlElems(pInfo, TSDB_SQL_CREATE_DNODE, 2, &X, &Y);}
cmd ::= CREATE DNODE IPTOKEN(X) PORT ids(Y). { setDCLSqlElems(pInfo, TSDB_SQL_CREATE_DNODE, 2, &X, &Y);}
cmd ::= CREATE ACCOUNT ids(X) PASS ids(Y) acct_optr(Z).
{ setCreateAcctSql(pInfo, TSDB_SQL_CREATE_ACCT, &X, &Y, &Z);}
cmd ::= CREATE DATABASE ifnotexists(Z) ids(X) db_optr(Y). { setCreateDbInfo(pInfo, TSDB_SQL_CREATE_DB, &X, &Y, &Z);}
@ -445,6 +446,15 @@ tagitemlist1(A) ::= tagitem1(Y). { A = taosArrayInit(4, sizeof(SToken)); taosArr
%type tagitem1 {SToken}
tagitem1(A) ::= MINUS(X) INTEGER(Y). { A.n = X.n + Y.n; A.type = Y.type; }
tagitem1(A) ::= MINUS(X) FLOAT(Y). { A.n = X.n + Y.n; A.type = Y.type; }
tagitem1(A) ::= PLUS(X) INTEGER(Y). { A.n = X.n + Y.n; A.type = Y.type; }
tagitem1(A) ::= PLUS(X) FLOAT(Y). { A.n = X.n + Y.n; A.type = Y.type; }
tagitem1(A) ::= INTEGER(X). { A = X; }
tagitem1(A) ::= FLOAT(X). { A = X; }
tagitem1(A) ::= STRING(X). { A = X; }
tagitem1(A) ::= BOOL(X). { A = X; }
tagitem1(A) ::= NULL(X). { A = X; }
tagitem1(A) ::= NOW(X). { A = X; }
%type tagitemlist {SArray*}
%destructor tagitemlist {taosArrayDestroy($$);}

229
source/libs/parser/inc/ttokendef.h
View File

@ -1,229 +0,0 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_TTOKENDEF_H
#define TDENGINE_TTOKENDEF_H
#define TK_ID 1
#define TK_BOOL 2
#define TK_TINYINT 3
#define TK_SMALLINT 4
#define TK_INTEGER 5
#define TK_BIGINT 6
#define TK_FLOAT 7
#define TK_DOUBLE 8
#define TK_STRING 9
#define TK_TIMESTAMP 10
#define TK_BINARY 11
#define TK_NCHAR 12
#define TK_OR 13
#define TK_AND 14
#define TK_NOT 15
#define TK_EQ 16
#define TK_NE 17
#define TK_ISNULL 18
#define TK_NOTNULL 19
#define TK_IS 20
#define TK_LIKE 21
#define TK_MATCH 22
#define TK_NMATCH 23
#define TK_GLOB 24
#define TK_BETWEEN 25
#define TK_IN 26
#define TK_GT 27
#define TK_GE 28
#define TK_LT 29
#define TK_LE 30
#define TK_BITAND 31
#define TK_BITOR 32
#define TK_LSHIFT 33
#define TK_RSHIFT 34
#define TK_PLUS 35
#define TK_MINUS 36
#define TK_DIVIDE 37
#define TK_TIMES 38
#define TK_STAR 39
#define TK_SLASH 40
#define TK_REM 41
#define TK_CONCAT 42
#define TK_UMINUS 43
#define TK_UPLUS 44
#define TK_BITNOT 45
#define TK_SHOW 46
#define TK_DATABASES 47
#define TK_TOPICS 48
#define TK_FUNCTIONS 49
#define TK_MNODES 50
#define TK_DNODES 51
#define TK_ACCOUNTS 52
#define TK_USERS 53
#define TK_MODULES 54
#define TK_QUERIES 55
#define TK_CONNECTIONS 56
#define TK_STREAMS 57
#define TK_VARIABLES 58
#define TK_SCORES 59
#define TK_GRANTS 60
#define TK_VNODES 61
#define TK_DOT 62
#define TK_CREATE 63
#define TK_TABLE 64
#define TK_STABLE 65
#define TK_DATABASE 66
#define TK_TABLES 67
#define TK_STABLES 68
#define TK_VGROUPS 69
#define TK_DROP 70
#define TK_TOPIC 71
#define TK_FUNCTION 72
#define TK_DNODE 73
#define TK_USER 74
#define TK_ACCOUNT 75
#define TK_USE 76
#define TK_DESCRIBE 77
#define TK_DESC 78
#define TK_ALTER 79
#define TK_PASS 80
#define TK_PRIVILEGE 81
#define TK_LOCAL 82
#define TK_COMPACT 83
#define TK_LP 84
#define TK_RP 85
#define TK_IF 86
#define TK_EXISTS 87
#define TK_PORT 88
#define TK_AS 89
#define TK_OUTPUTTYPE 90
#define TK_AGGREGATE 91
#define TK_BUFSIZE 92
#define TK_PPS 93
#define TK_TSERIES 94
#define TK_DBS 95
#define TK_STORAGE 96
#define TK_QTIME 97
#define TK_CONNS 98
#define TK_STATE 99
#define TK_COMMA 100
#define TK_KEEP 101
#define TK_CACHE 102
#define TK_REPLICA 103
#define TK_QUORUM 104
#define TK_DAYS 105
#define TK_MINROWS 106
#define TK_MAXROWS 107
#define TK_BLOCKS 108
#define TK_CTIME 109
#define TK_WAL 110
#define TK_FSYNC 111
#define TK_COMP 112
#define TK_PRECISION 113
#define TK_UPDATE 114
#define TK_CACHELAST 115
#define TK_UNSIGNED 116
#define TK_TAGS 117
#define TK_USING 118
#define TK_NULL 119
#define TK_NOW 120
#define TK_SELECT 121
#define TK_UNION 122
#define TK_ALL 123
#define TK_DISTINCT 124
#define TK_FROM 125
#define TK_VARIABLE 126
#define TK_INTERVAL 127
#define TK_EVERY 128
#define TK_SESSION 129
#define TK_STATE_WINDOW 130
#define TK_FILL 131
#define TK_SLIDING 132
#define TK_ORDER 133
#define TK_BY 134
#define TK_ASC 135
#define TK_GROUP 136
#define TK_HAVING 137
#define TK_LIMIT 138
#define TK_OFFSET 139
#define TK_SLIMIT 140
#define TK_SOFFSET 141
#define TK_WHERE 142
#define TK_RESET 143
#define TK_QUERY 144
#define TK_SYNCDB 145
#define TK_ADD 146
#define TK_COLUMN 147
#define TK_MODIFY 148
#define TK_TAG 149
#define TK_CHANGE 150
#define TK_SET 151
#define TK_KILL 152
#define TK_CONNECTION 153
#define TK_STREAM 154
#define TK_COLON 155
#define TK_ABORT 156
#define TK_AFTER 157
#define TK_ATTACH 158
#define TK_BEFORE 159
#define TK_BEGIN 160
#define TK_CASCADE 161
#define TK_CLUSTER 162
#define TK_CONFLICT 163
#define TK_COPY 164
#define TK_DEFERRED 165
#define TK_DELIMITERS 166
#define TK_DETACH 167
#define TK_EACH 168
#define TK_END 169
#define TK_EXPLAIN 170
#define TK_FAIL 171
#define TK_FOR 172
#define TK_IGNORE 173
#define TK_IMMEDIATE 174
#define TK_INITIALLY 175
#define TK_INSTEAD 176
#define TK_KEY 177
#define TK_OF 178
#define TK_RAISE 179
#define TK_REPLACE 180
#define TK_RESTRICT 181
#define TK_ROW 182
#define TK_STATEMENT 183
#define TK_TRIGGER 184
#define TK_VIEW 185
#define TK_IPTOKEN 186
#define TK_SEMI 187
#define TK_NONE 188
#define TK_PREV 189
#define TK_LINEAR 190
#define TK_IMPORT 191
#define TK_TBNAME 192
#define TK_JOIN 193
#define TK_INSERT 194
#define TK_INTO 195
#define TK_VALUES 196
#define TK_SPACE 300
#define TK_COMMENT 301
#define TK_ILLEGAL 302
#define TK_HEX 303 // hex number 0x123
#define TK_OCT 304 // oct number
#define TK_BIN 305 // bin format data 0b111
#define TK_FILE 306
#define TK_QUESTION 307 // denoting the placeholder of "?",when invoking statement bind query
#endif

2
source/libs/parser/src/astGenerator.c
View File

@ -276,7 +276,7 @@ bool tSqlExprIsLeaf(tSqlExpr *pExpr) {
return (pExpr->pRight == NULL && pExpr->pLeft == NULL) &&
(pExpr->tokenId == 0 ||
(pExpr->tokenId == TK_ID) ||
(pExpr->tokenId >= TK_BOOL && pExpr->tokenId <= TK_NCHAR) ||
(pExpr->tokenId == TK_BOOL || pExpr->tokenId == TK_STRING || pExpr->tokenId == TK_FLOAT) ||
(pExpr->tokenId == TK_NULL) ||
(pExpr->tokenId == TK_SET));
}

7
source/libs/parser/src/astToMsg.c
View File

@ -388,7 +388,7 @@ SCreateDnodeMsg *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMs
}
SToken* id = taosArrayGet(pInfo->pMiscInfo->a, 0);
if (id->type != TK_ID) {
if (id->type != TK_ID && id->type != TK_IPTOKEN) {
buildInvalidOperationMsg(pMsgBuf, msg2);
return NULL;
}
@ -414,8 +414,8 @@ SCreateDnodeMsg *buildCreateDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMs
return NULL;
}
strncpy(pCreate->ep, id->z, id->n);
pCreate->port = val;
strncpy(pCreate->fqdn, id->z, id->n);
pCreate->port = htonl(val);
*len = sizeof(SCreateDnodeMsg);
return pCreate;
@ -428,6 +428,7 @@ SDropDnodeMsg *buildDropDnodeMsg(SSqlInfo* pInfo, int32_t* len, SMsgBuf* pMsgBuf
char* end = NULL;
SDropDnodeMsg * pDrop = (SDropDnodeMsg *)calloc(1, sizeof(SDropDnodeMsg));
pDrop->dnodeId = strtoll(pzName->z, &end, 10);
pDrop->dnodeId = htonl(pDrop->dnodeId);
*len = sizeof(SDropDnodeMsg);
if (end - pzName->z != pzName->n) {

97
source/libs/parser/src/dCDAstProcess.c
View File

@ -1,3 +1,4 @@
#include <tmsg.h>
#include <ttime.h>
#include "astToMsg.h"
#include "parserInt.h"
@ -283,7 +284,7 @@ int32_t doCheckForCreateTable(SSqlInfo* pInfo, SMsgBuf* pMsgBuf) {
return TSDB_CODE_SUCCESS;
}
int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf) {
int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx *pCtx, SMsgBuf* pMsgBuf, char** pOutput, int32_t* len, SEpSet* pEpSet) {
const char* msg1 = "invalid table name";
const char* msg2 = "tags number not matched";
const char* msg3 = "tag value too long";
@ -313,16 +314,18 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx *pCtx, SMsgBuf* p
return code;
}
code = tNameExtractFullName(&name, pCreateTableInfo->tagdata.name);
const char* pStableName = tNameGetTableName(&name);
SArray* pValList = pCreateTableInfo->pTagVals;
if (code != TSDB_CODE_SUCCESS) {
return code;
}
size_t valSize = taosArrayGetSize(pValList);
size_t numOfInputTag = taosArrayGetSize(pValList);
STableMeta* pSuperTableMeta = NULL;
char dbName[TSDB_DB_FNAME_LEN] = {0};
tNameGetFullDbName(&name, dbName);
catalogGetTableMeta(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, dbName, pStableName, &pSuperTableMeta);
assert(pSuperTableMeta != NULL);
// too long tag values will return invalid sql, not be truncated automatically
SSchema *pTagSchema = getTableTagSchema(pSuperTableMeta);
STableComInfo tinfo = getTableInfo(pSuperTableMeta);
@ -341,7 +344,7 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx *pCtx, SMsgBuf* p
pNameList = pCreateTableInfo->pTagNames;
nameSize = taosArrayGetSize(pNameList);
if (valSize != nameSize || schemaSize < valSize) {
if (numOfInputTag != nameSize || schemaSize < numOfInputTag) {
tdDestroyKVRowBuilder(&kvRowBuilder);
return buildInvalidOperationMsg(pMsgBuf, msg2);
}
@ -417,33 +420,36 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx *pCtx, SMsgBuf* p
}
}
} else {
if (schemaSize != valSize) {
if (schemaSize != numOfInputTag) {
tdDestroyKVRowBuilder(&kvRowBuilder);
return buildInvalidOperationMsg(pMsgBuf, msg2);
}
for (int32_t i = 0; i < valSize; ++i) {
for (int32_t i = 0; i < numOfInputTag; ++i) {
SSchema *pSchema = &pTagSchema[i];
SListItem *pItem = taosArrayGet(pValList, i);
SToken* pItem = taosArrayGet(pValList, i);
char tagVal[TSDB_MAX_TAGS_LEN];
if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) {
if (pItem->pVar.nLen > pSchema->bytes) {
if (pItem->n > pSchema->bytes) {
tdDestroyKVRowBuilder(&kvRowBuilder);
return buildInvalidOperationMsg(pMsgBuf, msg3);
}
} else if (pSchema->type == TSDB_DATA_TYPE_TIMESTAMP) {
if (pItem->pVar.nType == TSDB_DATA_TYPE_BINARY) {
// code = convertTimestampStrToInt64(&(pItem->pVar), tinfo.precision);
if (code != TSDB_CODE_SUCCESS) {
return buildInvalidOperationMsg(pMsgBuf, msg4);
}
} else if (pItem->pVar.nType == TSDB_DATA_TYPE_TIMESTAMP) {
pItem->pVar.i = convertTimePrecision(pItem->pVar.i, TSDB_TIME_PRECISION_NANO, tinfo.precision);
}
// if (pItem->pVar.nType == TSDB_DATA_TYPE_BINARY) {
//// code = convertTimestampStrToInt64(&(pItem->pVar), tinfo.precision);
// if (code != TSDB_CODE_SUCCESS) {
// return buildInvalidOperationMsg(pMsgBuf, msg4);
// }
// } else if (pItem->pVar.nType == TSDB_DATA_TYPE_TIMESTAMP) {
// pItem->pVar.i = convertTimePrecision(pItem->pVar.i, TSDB_TIME_PRECISION_NANO, tinfo.precision);
// }
}
code = taosVariantDump(&(pItem->pVar), tagVal, pSchema->type, true);
char* endPtr = NULL;
int64_t v = strtoll(pItem->z, &endPtr, 10);
*(int32_t*) tagVal = v;
// code = taosVariantDump(&(pItem->pVar), tagVal, pSchema->type, true);
// check again after the convert since it may be converted from binary to nchar.
if (pSchema->type == TSDB_DATA_TYPE_BINARY || pSchema->type == TSDB_DATA_TYPE_NCHAR) {
@ -468,33 +474,37 @@ int32_t doCheckForCreateCTable(SSqlInfo* pInfo, SParseBasicCtx *pCtx, SMsgBuf* p
if (row == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
tdSortKVRowByColIdx(row);
pTag->dataLen = kvRowLen(row);
if (pTag->data == NULL) {
pTag->data = malloc(pTag->dataLen);
tdSortKVRowByColIdx(row);
SName tableName = {0};
code = createSName(&tableName, &pCreateTableInfo->name, pCtx, pMsgBuf);
if (code != TSDB_CODE_SUCCESS) {
return code;
}
kvRowCpy(pTag->data, row);
free(row);
struct SVCreateTbReq req = {0};
req.type = TD_CHILD_TABLE;
req.name = strdup(tNameGetTableName(&tableName));
req.ctbCfg.suid = pSuperTableMeta->suid;
req.ctbCfg.pTag = row;
bool dbIncluded2 = false;
// table name
// if (tscValidateName(&(pCreateTableInfo->name), true, &dbIncluded2) != TSDB_CODE_SUCCESS) {
// return buildInvalidOperationMsg(pMsgBuf, msg1);
// }
int32_t serLen = tSerializeSVCreateTbReq(NULL, &req);
char* buf1 = calloc(1, serLen);
char* p = buf1;
tSerializeSVCreateTbReq((void*) &buf1, &req);
*pOutput = p;
*len = serLen;
// STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, TABLE_INDEX);
// code = tscSetTableFullName(&pTableMetaInfo->name, &pCreateTableInfo->name, pSql, dbIncluded2);
// if (code != TSDB_CODE_SUCCESS) {
// return code;
// }
SVgroupInfo info = {0};
catalogGetTableHashVgroup(pCtx->pCatalog, pCtx->pTransporter, &pCtx->mgmtEpSet, dbName, req.name, &info);
// pCreateTableInfo->fullname = calloc(1, tNameLen(&pTableMetaInfo->name) + 1);
// code = tNameExtractFullName(&pTableMetaInfo->name, pCreateTableInfo->fullname);
// if (code != TSDB_CODE_SUCCESS) {
// return buildInvalidOperationMsg(pMsgBuf, msg1);
// }
pEpSet->inUse = info.inUse;
pEpSet->numOfEps = info.numOfEps;
for(int32_t i = 0; i < pEpSet->numOfEps; ++i) {
pEpSet->port[i] = info.epAddr[i].port;
tstrncpy(pEpSet->fqdn[i], info.epAddr[i].fqdn, tListLen(pEpSet->fqdn[i]));
}
}
return TSDB_CODE_SUCCESS;
@ -691,10 +701,11 @@ int32_t qParserValidateDclSqlNode(SSqlInfo* pInfo, SParseBasicCtx* pCtx, SDclStm
pDcl->pMsg = (char*)buildCreateTableMsg(pCreateTable, &pDcl->msgLen, pCtx, pMsgBuf);
pDcl->msgType = (pCreateTable->type == TSQL_CREATE_TABLE)? TDMT_VND_CREATE_TABLE:TDMT_MND_CREATE_STB;
} else if (pCreateTable->type == TSQL_CREATE_CTABLE) {
if ((code = doCheckForCreateCTable(pInfo, pCtx, pMsgBuf)) != TSDB_CODE_SUCCESS) {
if ((code = doCheckForCreateCTable(pInfo, pCtx, pMsgBuf, &pDcl->pMsg, &pDcl->msgLen, &pDcl->epSet)) != TSDB_CODE_SUCCESS) {
return code;
}
pDcl->msgType = TDMT_VND_CREATE_TABLE;
} else if (pCreateTable->type == TSQL_CREATE_STREAM) {
// if ((code = doCheckForStream(pSql, pInfo)) != TSDB_CODE_SUCCESS) {
// return code;

8
source/libs/parser/src/insertParser.c
View File

@ -162,7 +162,7 @@ static int32_t buildName(SInsertParseContext* pCxt, SToken* pStname, char* fullD
strncpy(fullDbName + n, pStname->z, p - pStname->z);
strncpy(tableName, p + 1, pStname->n - (p - pStname->z) - 1);
} else {
snprintf(fullDbName, TSDB_FULL_DB_NAME_LEN, "%d.%s", pCxt->pComCxt->ctx.acctId, pCxt->pComCxt->ctx.db);
snprintf(fullDbName, TSDB_DB_FNAME_LEN, "%d.%s", pCxt->pComCxt->ctx.acctId, pCxt->pComCxt->ctx.db);
strncpy(tableName, pStname->z, pStname->n);
}
@ -170,12 +170,12 @@ static int32_t buildName(SInsertParseContext* pCxt, SToken* pStname, char* fullD
}
static int32_t getTableMeta(SInsertParseContext* pCxt, SToken* pTname) {
char fullDbName[TSDB_FULL_DB_NAME_LEN] = {0};
char fullDbName[TSDB_DB_FNAME_LEN] = {0};
char tableName[TSDB_TABLE_NAME_LEN] = {0};
CHECK_CODE(buildName(pCxt, pTname, fullDbName, tableName));
CHECK_CODE(catalogGetTableMeta(pCxt->pComCxt->pCatalog, pCxt->pComCxt->pRpc, pCxt->pComCxt->pEpSet, fullDbName, tableName, &pCxt->pTableMeta));
CHECK_CODE(catalogGetTableMeta(pCxt->pComCxt->ctx.pCatalog, pCxt->pComCxt->ctx.pTransporter, &pCxt->pComCxt->ctx.mgmtEpSet, fullDbName, tableName, &pCxt->pTableMeta));
SVgroupInfo vg;
CHECK_CODE(catalogGetTableHashVgroup(pCxt->pComCxt->pCatalog, pCxt->pComCxt->pRpc, pCxt->pComCxt->pEpSet, fullDbName, tableName, &vg));
CHECK_CODE(catalogGetTableHashVgroup(pCxt->pComCxt->ctx.pCatalog, pCxt->pComCxt->ctx.pTransporter, &pCxt->pComCxt->ctx.mgmtEpSet, fullDbName, tableName, &vg));
CHECK_CODE(taosHashPut(pCxt->pVgroupsHashObj, (const char*)&vg.vgId, sizeof(vg.vgId), (char*)&vg, sizeof(vg)));
return TSDB_CODE_SUCCESS;
}

2
source/libs/parser/src/parser.c
View File

@ -229,6 +229,6 @@ void qParserClearupMetaRequestInfo(SCatalogReq* pMetaReq) {
taosArrayDestroy(pMetaReq->pUdf);
}
void qDestoryQuery(SQueryNode* pQuery) {
void qDestroyQuery(SQueryNode* pQuery) {
// todo
}

2
source/libs/parser/src/parserUtil.c
View File

@ -1918,7 +1918,7 @@ char* cloneCurrentDBName(SSqlObj* pSql) {
case TAOS_REQ_FROM_HTTP:
pCtx = pSql->param;
if (pCtx && pCtx->db[0] != '\0') {
char db[TSDB_FULL_DB_NAME_LEN] = {0};
char db[TSDB_DB_FNAME_LEN] = {0};
int32_t len = sprintf(db, "%s%s%s", pTscObj->acctId, TS_PATH_DELIMITER, pCtx->db);
assert(len <= sizeof(db));

2
source/libs/parser/src/queryInfoUtil.c
View File

@ -26,7 +26,7 @@ size_t getNumOfExprs(SQueryStmtInfo* pQueryInfo) {
}
SSchema* getOneColumnSchema(const STableMeta* pTableMeta, int32_t colIndex) {
assert(pTableMeta != NULL && pTableMeta->schema != NULL && colIndex >= 0 && colIndex < getNumOfColumns(pTableMeta));
assert(pTableMeta != NULL && pTableMeta->schema != NULL && colIndex >= 0 && colIndex < (getNumOfColumns(pTableMeta) + getNumOfTags(pTableMeta)));
SSchema* pSchema = (SSchema*) pTableMeta->schema;
return &pSchema[colIndex];

3262
source/libs/parser/src/sql.c
View File

File diff suppressed because it is too large Load Diff

14
source/libs/parser/src/ttokenizer.c
View File

@ -31,17 +31,17 @@ typedef struct SKeyword {
static SKeyword keywordTable[] = {
{"ID", TK_ID},
{"BOOL", TK_BOOL},
{"TINYINT", TK_TINYINT},
{"SMALLINT", TK_SMALLINT},
// {"TINYINT", TK_TINYINT},
// {"SMALLINT", TK_SMALLINT},
{"INTEGER", TK_INTEGER},
{"INT", TK_INTEGER},
{"BIGINT", TK_BIGINT},
// {"BIGINT", TK_BIGINT},
{"FLOAT", TK_FLOAT},
{"DOUBLE", TK_DOUBLE},
// {"DOUBLE", TK_DOUBLE},
{"STRING", TK_STRING},
{"TIMESTAMP", TK_TIMESTAMP},
{"BINARY", TK_BINARY},
{"NCHAR", TK_NCHAR},
// {"BINARY", TK_BINARY},
// {"NCHAR", TK_NCHAR},
{"OR", TK_OR},
{"AND", TK_AND},
{"NOT", TK_NOT},
@ -672,7 +672,7 @@ SToken tStrGetToken(const char* str, int32_t* i, bool isPrevOptr) {
}
}
t0.z = str + (*i);
t0.z = (char*) str + (*i);
*i += t0.n;
return t0;

4
source/libs/parser/test/parserTests.cpp
View File

@ -715,7 +715,7 @@ TEST(testCase, show_user_Test) {
ASSERT_EQ(info1.valid, true);
SDclStmtInfo output;
SParseBasicCtx ct= {.db= "abc", .acctId = 1, .requestId = 1};
SParseBasicCtx ct= {.requestId = 1, .acctId = 1, .db = "abc", .pTransporter = NULL};
int32_t code = qParserValidateDclSqlNode(&info1, &ct, &output, msg, buf.len);
ASSERT_EQ(code, 0);
@ -736,7 +736,7 @@ TEST(testCase, create_user_Test) {
ASSERT_EQ(isDclSqlStatement(&info1), true);
SDclStmtInfo output;
SParseBasicCtx ct= {.db= "abc", .acctId = 1, .requestId = 1};
SParseBasicCtx ct= {.requestId = 1, .acctId = 1, .db = "abc"};
int32_t code = qParserValidateDclSqlNode(&info1, &ct, &output, msg, buf.len);
ASSERT_EQ(code, 0);

81
source/libs/planner/src/physicalPlanJson.c
View File

@ -857,3 +857,84 @@ int32_t stringToSubplan(const char* str, SSubplan** subplan) {
*subplan = subplanFromJson(json);
return (NULL == *subplan ? TSDB_CODE_FAILED : TSDB_CODE_SUCCESS);
}
cJSON* qDagToJson(const SQueryDag* pDag) {
cJSON* pRoot = cJSON_CreateObject();
if(pRoot == NULL) {
return NULL;
}
cJSON_AddNumberToObject(pRoot, "numOfSubplans", pDag->numOfSubplans);
cJSON_AddNumberToObject(pRoot, "queryId", pDag->queryId);
cJSON *pLevels = cJSON_CreateArray();
if(pLevels == NULL) {
cJSON_Delete(pRoot);
return NULL;
}
cJSON_AddItemToObject(pRoot, "pSubplans", pLevels);
size_t level = taosArrayGetSize(pDag->pSubplans);
for(size_t i = 0; i < level; i++) {
const SArray* pSubplans = (const SArray*)taosArrayGetP(pDag->pSubplans, i);
size_t num = taosArrayGetSize(pSubplans);
cJSON* plansOneLevel = cJSON_CreateArray();
if(plansOneLevel == NULL) {
cJSON_Delete(pRoot);
return NULL;
}
cJSON_AddItemToArray(pLevels, plansOneLevel);
for(size_t j = 0; j < num; j++) {
cJSON* pSubplan = subplanToJson((const SSubplan*)taosArrayGetP(pSubplans, j));
if(pSubplan == NULL) {
cJSON_Delete(pRoot);
return NULL;
}
cJSON_AddItemToArray(plansOneLevel, pSubplan);
}
}
return pRoot;
}
char* qDagToString(const SQueryDag* pDag) {
cJSON* pRoot = qDagToJson(pDag);
return cJSON_Print(pRoot);
}
SQueryDag* qJsonToDag(const cJSON* pRoot) {
SQueryDag* pDag = malloc(sizeof(SQueryDag));
if(pDag == NULL) {
return NULL;
}
pDag->numOfSubplans = cJSON_GetNumberValue(cJSON_GetObjectItem(pRoot, "numOfSubplans"));
pDag->queryId = cJSON_GetNumberValue(cJSON_GetObjectItem(pRoot, "queryId"));
pDag->pSubplans = taosArrayInit(0, sizeof(SArray));
if (pDag->pSubplans == NULL) {
free(pDag);
return NULL;
}
cJSON* pLevels = cJSON_GetObjectItem(pRoot, "pSubplans");
int level = cJSON_GetArraySize(pLevels);
for(int i = 0; i < level; i++) {
SArray* plansOneLevel = taosArrayInit(0, sizeof(void*));
if(plansOneLevel == NULL) {
for(int j = 0; j < i; j++) {
taosArrayDestroy(taosArrayGetP(pDag->pSubplans, j));
}
taosArrayDestroy(pDag->pSubplans);
free(pDag);
return NULL;
}
cJSON* pItem = cJSON_GetArrayItem(pLevels, i);
int sz = cJSON_GetArraySize(pItem);
for(int j = 0; j < sz; j++) {
cJSON* pSubplanJson = cJSON_GetArrayItem(pItem, j);
SSubplan* pSubplan = subplanFromJson(pSubplanJson);
taosArrayPush(plansOneLevel, &pSubplan);
}
taosArrayPush(pDag->pSubplans, plansOneLevel);
}
return pDag;
}
SQueryDag* qStringToDag(const char* pStr) {
cJSON* pRoot = cJSON_Parse(pStr);
return qJsonToDag(pRoot);
}

15
source/libs/planner/test/phyPlanTests.cpp
View File

@ -83,7 +83,7 @@ protected:
}
}
SQueryDag* reslut() {
SQueryDag* result() {
return dag_.get();
}
@ -149,16 +149,23 @@ TEST_F(PhyPlanTest, tableScanTest) {
pushScan("test", "t1", QNODE_TABLESCAN);
ASSERT_EQ(run(), TSDB_CODE_SUCCESS);
explain();
SQueryDag* dag = reslut();
SQueryDag* dag = result();
// todo check
}
TEST_F(PhyPlanTest, serializeTest) {
pushScan("test", "t1", QNODE_TABLESCAN);
ASSERT_EQ(run(), TSDB_CODE_SUCCESS);
SQueryDag* dag = result();
cout << qDagToString(dag) << endl;
}
// select * from supertable
TEST_F(PhyPlanTest, superTableScanTest) {
pushScan("test", "st1", QNODE_TABLESCAN);
ASSERT_EQ(run(), TSDB_CODE_SUCCESS);
explain();
SQueryDag* dag = reslut();
SQueryDag* dag = result();
// todo check
}
@ -166,6 +173,6 @@ TEST_F(PhyPlanTest, superTableScanTest) {
TEST_F(PhyPlanTest, insertTest) {
ASSERT_EQ(run("test", "insert into t1 values (now, 1, \"beijing\")"), TSDB_CODE_SUCCESS);
explain();
SQueryDag* dag = reslut();
SQueryDag* dag = result();
// todo check
}

15
source/libs/qcom/src/querymsg.c
View File

@ -16,6 +16,7 @@
#include "tmsg.h"
#include "queryInt.h"
#include "query.h"
#include "trpc.h"
int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char **msg, int32_t msgSize, int32_t *msgLen) = {0};
@ -31,7 +32,7 @@ int32_t queryBuildTableMetaReqMsg(void* input, char **msg, int32_t msgSize, int3
int32_t estimateSize = sizeof(STableInfoMsg);
if (NULL == *msg || msgSize < estimateSize) {
tfree(*msg);
*msg = calloc(1, estimateSize);
*msg = rpcMallocCont(estimateSize);
if (NULL == *msg) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
@ -59,7 +60,7 @@ int32_t queryBuildUseDbMsg(void* input, char **msg, int32_t msgSize, int32_t *ms
int32_t estimateSize = sizeof(SUseDbMsg);
if (NULL == *msg || msgSize < estimateSize) {
tfree(*msg);
*msg = calloc(1, estimateSize);
*msg = rpcMallocCont(estimateSize);
if (NULL == *msg) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
@ -265,11 +266,13 @@ int32_t queryProcessTableMetaRsp(void* output, char *msg, int32_t msgSize) {
void initQueryModuleMsgHandle() {
queryBuildMsg[TDMT_VND_TABLE_META] = queryBuildTableMetaReqMsg;
queryBuildMsg[TDMT_MND_USE_DB] = queryBuildUseDbMsg;
queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryBuildTableMetaReqMsg;
queryBuildMsg[TMSG_INDEX(TDMT_MND_STB_META)] = queryBuildTableMetaReqMsg;
queryBuildMsg[TMSG_INDEX(TDMT_MND_USE_DB)] = queryBuildUseDbMsg;
queryProcessMsgRsp[TDMT_VND_TABLE_META] = queryProcessTableMetaRsp;
queryProcessMsgRsp[TDMT_MND_USE_DB] = queryProcessUseDBRsp;
queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryProcessTableMetaRsp;
queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_STB_META)] = queryProcessTableMetaRsp;
queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_USE_DB)] = queryProcessUseDBRsp;
}

53
source/libs/qworker/inc/qworkerInt.h
View File

@ -42,25 +42,41 @@ enum {
QW_WRITE,
};
typedef struct SQWorkerTaskStatus {
enum {
QW_EXIST_ACQUIRE = 1,
QW_EXIST_RET_ERR,
};
enum {
QW_NOT_EXIST_RET_ERR = 1,
QW_NOT_EXIST_ADD,
};
enum {
QW_ADD_RET_ERR = 1,
QW_ADD_ACQUIRE,
};
typedef struct SQWTaskStatus {
SRWLatch lock;
int32_t code;
int8_t status;
int8_t ready;
bool cancel;
bool drop;
} SQWorkerTaskStatus;
} SQWTaskStatus;
typedef struct SQWorkerResCache {
SRWLatch lock;
void *data;
} SQWorkerResCache;
typedef struct SQWorkerSchStatus {
typedef struct SQWSchStatus {
int32_t lastAccessTs; // timestamp in second
SRWLatch tasksLock;
SHashObj *tasksHash; // key:queryId+taskId, value: SQWorkerTaskStatus
} SQWorkerSchStatus;
} SQWSchStatus;
// Qnode/Vnode level task management
typedef struct SQWorkerMgmt {
@ -71,7 +87,7 @@ typedef struct SQWorkerMgmt {
SHashObj *resHash; //key: queryId+taskId, value: SQWorkerResCache
} SQWorkerMgmt;
#define QW_GOT_RES_DATA(data) (false)
#define QW_GOT_RES_DATA(data) (true)
#define QW_LOW_RES_DATA(data) (false)
#define QW_TASK_NOT_EXIST(code) (TSDB_CODE_QRY_SCH_NOT_EXIST == (code) || TSDB_CODE_QRY_TASK_NOT_EXIST == (code))
@ -86,8 +102,31 @@ typedef struct SQWorkerMgmt {
#define QW_ERR_LRET(c,...) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { qError(__VA_ARGS__); terrno = _code; return _code; } } while (0)
#define QW_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0)
#define QW_LOCK(type, _lock) (QW_READ == (type) ? taosRLockLatch(_lock) : taosWLockLatch(_lock))
#define QW_UNLOCK(type, _lock) (QW_READ == (type) ? taosRUnLockLatch(_lock) : taosWUnLockLatch(_lock))
#define QW_LOCK(type, _lock) do { \
if (QW_READ == (type)) { \
if ((*(_lock)) < 0) assert(0); \
taosRLockLatch(_lock); \
qDebug("RLOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \
} else { \
if ((*(_lock)) < 0) assert(0); \
taosWLockLatch(_lock); \
qDebug("WLOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \
} \
} while (0)
#define QW_UNLOCK(type, _lock) do { \
if (QW_READ == (type)) { \
if ((*(_lock)) <= 0) assert(0); \
taosRUnLockLatch(_lock); \
qDebug("RULOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \
} else { \
if ((*(_lock)) <= 0) assert(0); \
taosWUnLockLatch(_lock); \
qDebug("WULOCK%p, %s:%d", (_lock), __FILE__, __LINE__); \
} \
} while (0)
static int32_t qwAcquireScheduler(int32_t rwType, SQWorkerMgmt *mgmt, uint64_t sId, SQWSchStatus **sch, int32_t nOpt);
#ifdef __cplusplus

587
source/libs/qworker/src/qworker.c
View File

File diff suppressed because it is too large Load Diff

368
source/libs/qworker/test/qworkerTests.cpp
View File

@ -36,10 +36,25 @@
namespace {
bool testStop = false;
int32_t qwtStringToPlan(const char* str, SSubplan** subplan) {
return 0;
}
void qwtRpcSendResponse(const SRpcMsg *pRsp) {
if (TDMT_VND_TASKS_STATUS_RSP == pRsp->msgType) {
SSchedulerStatusRsp *rsp = (SSchedulerStatusRsp *)pRsp->pCont;
printf("task num:%d\n", rsp->num);
for (int32_t i = 0; i < rsp->num; ++i) {
STaskStatus *task = &rsp->status[i];
printf("qId:%"PRIx64",tId:%"PRIx64",status:%d\n", task->queryId, task->taskId, task->status);
}
}
return;
}
void stubSetStringToPlan() {
static Stub stub;
@ -54,11 +69,148 @@ void stubSetStringToPlan() {
}
}
void stubSetRpcSendResponse() {
static Stub stub;
stub.set(rpcSendResponse, qwtRpcSendResponse);
{
AddrAny any("libplanner.so");
std::map<std::string,void*> result;
any.get_global_func_addr_dynsym("^rpcSendResponse$", result);
for (const auto& f : result) {
stub.set(f.second, qwtRpcSendResponse);
}
}
}
void *queryThread(void *param) {
SRpcMsg queryRpc = {0};
int32_t code = 0;
uint32_t n = 0;
void *mockPointer = (void *)0x1;
void *mgmt = param;
SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100);
queryMsg->queryId = htobe64(1);
queryMsg->sId = htobe64(1);
queryMsg->taskId = htobe64(1);
queryMsg->contentLen = htonl(100);
queryRpc.pCont = queryMsg;
queryRpc.contLen = sizeof(SSubQueryMsg) + 100;
while (!testStop) {
qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc);
usleep(rand()%5);
if (++n % 50000 == 0) {
printf("query:%d\n", n);
}
}
return NULL;
}
void *readyThread(void *param) {
SRpcMsg readyRpc = {0};
int32_t code = 0;
uint32_t n = 0;
void *mockPointer = (void *)0x1;
void *mgmt = param;
SResReadyMsg readyMsg = {0};
readyMsg.sId = htobe64(1);
readyMsg.queryId = htobe64(1);
readyMsg.taskId = htobe64(1);
readyRpc.pCont = &readyMsg;
readyRpc.contLen = sizeof(SResReadyMsg);
while (!testStop) {
code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc);
usleep(rand()%5);
if (++n % 50000 == 0) {
printf("ready:%d\n", n);
}
}
return NULL;
}
void *fetchThread(void *param) {
SRpcMsg fetchRpc = {0};
int32_t code = 0;
uint32_t n = 0;
void *mockPointer = (void *)0x1;
void *mgmt = param;
SResFetchMsg fetchMsg = {0};
fetchMsg.sId = htobe64(1);
fetchMsg.queryId = htobe64(1);
fetchMsg.taskId = htobe64(1);
fetchRpc.pCont = &fetchMsg;
fetchRpc.contLen = sizeof(SResFetchMsg);
while (!testStop) {
code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc);
usleep(rand()%5);
if (++n % 50000 == 0) {
printf("fetch:%d\n", n);
}
}
return NULL;
}
void *dropThread(void *param) {
SRpcMsg dropRpc = {0};
int32_t code = 0;
uint32_t n = 0;
void *mockPointer = (void *)0x1;
void *mgmt = param;
STaskDropMsg dropMsg = {0};
dropMsg.sId = htobe64(1);
dropMsg.queryId = htobe64(1);
dropMsg.taskId = htobe64(1);
dropRpc.pCont = &dropMsg;
dropRpc.contLen = sizeof(STaskDropMsg);
while (!testStop) {
code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc);
usleep(rand()%5);
if (++n % 50000 == 0) {
printf("drop:%d\n", n);
}
}
return NULL;
}
void *statusThread(void *param) {
SRpcMsg statusRpc = {0};
int32_t code = 0;
uint32_t n = 0;
void *mockPointer = (void *)0x1;
void *mgmt = param;
SSchTasksStatusMsg statusMsg = {0};
statusMsg.sId = htobe64(1);
statusRpc.pCont = &statusMsg;
statusRpc.contLen = sizeof(SSchTasksStatusMsg);
statusRpc.msgType = TDMT_VND_TASKS_STATUS;
while (!testStop) {
statusMsg.sId = htobe64(1);
code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc);
usleep(rand()%5);
if (++n % 50000 == 0) {
printf("status:%d\n", n);
}
}
return NULL;
}
}
TEST(testCase, normalCase) {
TEST(seqTest, normalCase) {
void *mgmt = NULL;
int32_t code = 0;
void *mockPointer = (void *)0x1;
@ -66,48 +218,254 @@ TEST(testCase, normalCase) {
SRpcMsg readyRpc = {0};
SRpcMsg fetchRpc = {0};
SRpcMsg dropRpc = {0};
SRpcMsg statusRpc = {0};
SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100);
queryMsg->queryId = htobe64(1);
queryMsg->schedulerId = htobe64(1);
queryMsg->sId = htobe64(1);
queryMsg->taskId = htobe64(1);
queryMsg->contentLen = htonl(100);
queryRpc.pCont = queryMsg;
queryRpc.contLen = sizeof(SSubQueryMsg) + 100;
SResReadyMsg readyMsg = {0};
readyMsg.schedulerId = htobe64(1);
readyMsg.sId = htobe64(1);
readyMsg.queryId = htobe64(1);
readyMsg.taskId = htobe64(1);
readyRpc.pCont = &readyMsg;
readyRpc.contLen = sizeof(SResReadyMsg);
SResFetchMsg fetchMsg = {0};
fetchMsg.schedulerId = htobe64(1);
fetchMsg.sId = htobe64(1);
fetchMsg.queryId = htobe64(1);
fetchMsg.taskId = htobe64(1);
fetchRpc.pCont = &fetchMsg;
fetchRpc.contLen = sizeof(SResFetchMsg);
STaskDropMsg dropMsg = {0};
dropMsg.schedulerId = htobe64(1);
dropMsg.sId = htobe64(1);
dropMsg.queryId = htobe64(1);
dropMsg.taskId = htobe64(1);
dropRpc.pCont = &dropMsg;
dropRpc.contLen = sizeof(STaskDropMsg);
SSchTasksStatusMsg statusMsg = {0};
statusMsg.sId = htobe64(1);
statusRpc.pCont = &statusMsg;
statusRpc.contLen = sizeof(SSchTasksStatusMsg);
statusRpc.msgType = TDMT_VND_TASKS_STATUS;
stubSetStringToPlan();
stubSetRpcSendResponse();
code = qWorkerInit(NULL, &mgmt);
ASSERT_EQ(code, 0);
statusMsg.sId = htobe64(1);
code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc);
ASSERT_EQ(code, 0);
code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc);
ASSERT_EQ(code, 0);
statusMsg.sId = htobe64(1);
code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc);
ASSERT_EQ(code, 0);
code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc);
ASSERT_EQ(code, 0);
statusMsg.sId = htobe64(1);
code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc);
ASSERT_EQ(code, 0);
code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc);
ASSERT_EQ(code, 0);
statusMsg.sId = htobe64(1);
code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc);
ASSERT_EQ(code, 0);
code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc);
ASSERT_EQ(code, 0);
statusMsg.sId = htobe64(1);
code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc);
ASSERT_EQ(code, 0);
qWorkerDestroy(&mgmt);
}
TEST(seqTest, cancelFirst) {
void *mgmt = NULL;
int32_t code = 0;
void *mockPointer = (void *)0x1;
SRpcMsg queryRpc = {0};
SRpcMsg dropRpc = {0};
SRpcMsg statusRpc = {0};
SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100);
queryMsg->queryId = htobe64(1);
queryMsg->sId = htobe64(1);
queryMsg->taskId = htobe64(1);
queryMsg->contentLen = htonl(100);
queryRpc.pCont = queryMsg;
queryRpc.contLen = sizeof(SSubQueryMsg) + 100;
STaskDropMsg dropMsg = {0};
dropMsg.sId = htobe64(1);
dropMsg.queryId = htobe64(1);
dropMsg.taskId = htobe64(1);
dropRpc.pCont = &dropMsg;
dropRpc.contLen = sizeof(STaskDropMsg);
SSchTasksStatusMsg statusMsg = {0};
statusMsg.sId = htobe64(1);
statusRpc.pCont = &statusMsg;
statusRpc.contLen = sizeof(SSchTasksStatusMsg);
statusRpc.msgType = TDMT_VND_TASKS_STATUS;
stubSetStringToPlan();
stubSetRpcSendResponse();
code = qWorkerInit(NULL, &mgmt);
ASSERT_EQ(code, 0);
statusMsg.sId = htobe64(1);
code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc);
ASSERT_EQ(code, 0);
code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc);
ASSERT_EQ(code, 0);
statusMsg.sId = htobe64(1);
code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc);
ASSERT_EQ(code, 0);
code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc);
ASSERT_EQ(code, 0);
statusMsg.sId = htobe64(1);
code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc);
ASSERT_EQ(code, 0);
qWorkerDestroy(&mgmt);
}
TEST(seqTest, randCase) {
void *mgmt = NULL;
int32_t code = 0;
void *mockPointer = (void *)0x1;
SRpcMsg queryRpc = {0};
SRpcMsg readyRpc = {0};
SRpcMsg fetchRpc = {0};
SRpcMsg dropRpc = {0};
SRpcMsg statusRpc = {0};
SSubQueryMsg *queryMsg = (SSubQueryMsg *)calloc(1, sizeof(SSubQueryMsg) + 100);
queryMsg->queryId = htobe64(1);
queryMsg->sId = htobe64(1);
queryMsg->taskId = htobe64(1);
queryMsg->contentLen = htonl(100);
queryRpc.pCont = queryMsg;
queryRpc.contLen = sizeof(SSubQueryMsg) + 100;
SResReadyMsg readyMsg = {0};
readyMsg.sId = htobe64(1);
readyMsg.queryId = htobe64(1);
readyMsg.taskId = htobe64(1);
readyRpc.pCont = &readyMsg;
readyRpc.contLen = sizeof(SResReadyMsg);
SResFetchMsg fetchMsg = {0};
fetchMsg.sId = htobe64(1);
fetchMsg.queryId = htobe64(1);
fetchMsg.taskId = htobe64(1);
fetchRpc.pCont = &fetchMsg;
fetchRpc.contLen = sizeof(SResFetchMsg);
STaskDropMsg dropMsg = {0};
dropMsg.sId = htobe64(1);
dropMsg.queryId = htobe64(1);
dropMsg.taskId = htobe64(1);
dropRpc.pCont = &dropMsg;
dropRpc.contLen = sizeof(STaskDropMsg);
SSchTasksStatusMsg statusMsg = {0};
statusMsg.sId = htobe64(1);
statusRpc.pCont = &statusMsg;
statusRpc.contLen = sizeof(SSchTasksStatusMsg);
statusRpc.msgType = TDMT_VND_TASKS_STATUS;
stubSetStringToPlan();
stubSetRpcSendResponse();
srand(time(NULL));
code = qWorkerInit(NULL, &mgmt);
ASSERT_EQ(code, 0);
int32_t t = 0;
int32_t maxr = 10001;
while (true) {
int32_t r = rand() % maxr;
if (r >= 0 && r < maxr/5) {
printf("Query,%d\n", t++);
code = qWorkerProcessQueryMsg(mockPointer, mgmt, &queryRpc);
} else if (r >= maxr/5 && r < maxr * 2/5) {
printf("Ready,%d\n", t++);
code = qWorkerProcessReadyMsg(mockPointer, mgmt, &readyRpc);
} else if (r >= maxr * 2/5 && r < maxr* 3/5) {
printf("Fetch,%d\n", t++);
code = qWorkerProcessFetchMsg(mockPointer, mgmt, &fetchRpc);
} else if (r >= maxr * 3/5 && r < maxr * 4/5) {
printf("Drop,%d\n", t++);
code = qWorkerProcessDropMsg(mockPointer, mgmt, &dropRpc);
} else if (r >= maxr * 4/5 && r < maxr-1) {
printf("Status,%d\n", t++);
statusMsg.sId = htobe64(1);
code = qWorkerProcessStatusMsg(mockPointer, mgmt, &statusRpc);
ASSERT_EQ(code, 0);
} else {
printf("QUIT RAND NOW");
break;
}
}
qWorkerDestroy(&mgmt);
}
TEST(seqTest, multithreadRand) {
void *mgmt = NULL;
int32_t code = 0;
void *mockPointer = (void *)0x1;
stubSetStringToPlan();
stubSetRpcSendResponse();
srand(time(NULL));
code = qWorkerInit(NULL, &mgmt);
ASSERT_EQ(code, 0);
pthread_attr_t thattr;
pthread_attr_init(&thattr);
pthread_t t1,t2,t3,t4,t5;
pthread_create(&(t1), &thattr, queryThread, mgmt);
pthread_create(&(t2), &thattr, readyThread, NULL);
pthread_create(&(t3), &thattr, fetchThread, NULL);
pthread_create(&(t4), &thattr, dropThread, NULL);
pthread_create(&(t5), &thattr, statusThread, NULL);
int32_t t = 0;
int32_t maxr = 10001;
sleep(300);
testStop = true;
sleep(1);
qWorkerDestroy(&mgmt);
}

8
source/libs/scheduler/inc/schedulerInt.h
View File

@ -38,11 +38,16 @@ enum {
typedef struct SSchedulerMgmt {
uint64_t taskId;
uint64_t schedulerId;
uint64_t sId;
SSchedulerCfg cfg;
SHashObj *jobs; // key: queryId, value: SQueryJob*
} SSchedulerMgmt;
typedef struct SSchCallbackParam {
uint64_t queryId;
uint64_t taskId;
} SSchCallbackParam;
typedef struct SSchLevel {
int32_t level;
int8_t status;
@ -120,6 +125,7 @@ typedef struct SSchJob {
extern int32_t schLaunchTask(SSchJob *job, SSchTask *task);
extern int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, int32_t msgType);
#ifdef __cplusplus
}

356
source/libs/scheduler/src/scheduler.c
View File

@ -21,36 +21,6 @@
SSchedulerMgmt schMgmt = {0};
int32_t schBuildAndSendRequest(void *pRpc, const SEpSet* pMgmtEps, __taos_async_fn_t fp) {
/*
SRequestObj *pRequest = createRequest(pTscObj, fp, param, TSDB_SQL_CONNECT);
if (pRequest == NULL) {
return TSDB_CODE_TSC_OUT_OF_MEMORY;
}
SRequestMsgBody body = {0};
buildConnectMsg(pRequest, &body);
int64_t transporterId = 0;
asyncSendMsgToServer(pTscObj->pTransporter, &pTscObj->pAppInfo->mgmtEp.epSet, &body, &transporterId);
tsem_wait(&pRequest->body.rspSem);
destroyConnectMsg(&body);
if (pRequest->code != TSDB_CODE_SUCCESS) {
const char *errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(terrno);
printf("failed to connect to server, reason: %s\n\n", errorMsg);
destroyRequest(pRequest);
taos_close(pTscObj);
pTscObj = NULL;
} else {
tscDebug("0x%"PRIx64" connection is opening, connId:%d, dnodeConn:%p", pTscObj->id, pTscObj->connId, pTscObj->pTransporter);
destroyRequest(pRequest);
}
*/
}
int32_t schBuildTaskRalation(SSchJob *job, SHashObj *planToTask) {
for (int32_t i = 0; i < job->levelNum; ++i) {
SSchLevel *level = taosArrayGet(job->levels, i);
@ -312,100 +282,6 @@ int32_t schMoveTaskToFailList(SSchJob *job, SSchTask *task, bool *moved) {
return TSDB_CODE_SUCCESS;
}
int32_t schAsyncSendMsg(SSchJob *job, SSchTask *task, int32_t msgType) {
int32_t msgSize = 0;
void *msg = NULL;
switch (msgType) {
case TDMT_VND_SUBMIT: {
if (NULL == task->msg || task->msgLen <= 0) {
qError("submit msg is NULL");
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
msgSize = task->msgLen;
msg = task->msg;
break;
}
case TDMT_VND_QUERY: {
if (NULL == task->msg) {
qError("query msg is NULL");
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
msgSize = sizeof(SSubQueryMsg) + task->msgLen;
msg = calloc(1, msgSize);
if (NULL == msg) {
qError("calloc %d failed", msgSize);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
SSubQueryMsg *pMsg = msg;
pMsg->schedulerId = htobe64(schMgmt.schedulerId);
pMsg->queryId = htobe64(job->queryId);
pMsg->taskId = htobe64(task->taskId);
pMsg->contentLen = htonl(task->msgLen);
memcpy(pMsg->msg, task->msg, task->msgLen);
break;
}
case TDMT_VND_RES_READY: {
msgSize = sizeof(SResReadyMsg);
msg = calloc(1, msgSize);
if (NULL == msg) {
qError("calloc %d failed", msgSize);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
SResReadyMsg *pMsg = msg;
pMsg->schedulerId = htobe64(schMgmt.schedulerId);
pMsg->queryId = htobe64(job->queryId);
pMsg->taskId = htobe64(task->taskId);
break;
}
case TDMT_VND_FETCH: {
if (NULL == task) {
SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
msgSize = sizeof(SResFetchMsg);
msg = calloc(1, msgSize);
if (NULL == msg) {
qError("calloc %d failed", msgSize);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
SResFetchMsg *pMsg = msg;
pMsg->schedulerId = htobe64(schMgmt.schedulerId);
pMsg->queryId = htobe64(job->queryId);
pMsg->taskId = htobe64(task->taskId);
break;
}
case TDMT_VND_DROP_TASK:{
msgSize = sizeof(STaskDropMsg);
msg = calloc(1, msgSize);
if (NULL == msg) {
qError("calloc %d failed", msgSize);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
STaskDropMsg *pMsg = msg;
pMsg->schedulerId = htobe64(schMgmt.schedulerId);
pMsg->queryId = htobe64(job->queryId);
pMsg->taskId = htobe64(task->taskId);
break;
}
default:
qError("unknown msg type:%d", msgType);
break;
}
//TODO SEND MSG
//taosAsyncExec(rpcSendRequest(void * shandle, const SEpSet * pEpSet, SRpcMsg * pMsg, int64_t * pRid), p, &code);
return TSDB_CODE_SUCCESS;
}
int32_t schTaskCheckAndSetRetry(SSchJob *job, SSchTask *task, int32_t errCode, bool *needRetry) {
// TODO set retry or not based on task type/errCode/retry times/job status/available eps...
// TODO if needRetry, set task retry info
@ -424,7 +300,7 @@ int32_t schFetchFromRemote(SSchJob *job) {
return TSDB_CODE_SUCCESS;
}
SCH_ERR_JRET(schAsyncSendMsg(job, job->fetchTask, TDMT_VND_FETCH));
SCH_ERR_JRET(schBuildAndSendMsg(job, job->fetchTask, TDMT_VND_FETCH));
return TSDB_CODE_SUCCESS;
@ -577,11 +453,11 @@ int32_t schProcessOnTaskFailure(SSchJob *job, SSchTask *task, int32_t errCode) {
return TSDB_CODE_SUCCESS;
}
int32_t schHandleRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode) {
int32_t schProcessRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode) {
int32_t code = 0;
switch (msgType) {
case TDMT_VND_SUBMIT: {
case TDMT_VND_SUBMIT_RSP: {
SShellSubmitRspMsg *rsp = (SShellSubmitRspMsg *)msg;
if (rsp->code != TSDB_CODE_SUCCESS) {
SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rsp->code));
@ -595,20 +471,20 @@ int32_t schHandleRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg
}
break;
}
case TDMT_VND_QUERY: {
case TDMT_VND_QUERY_RSP: {
SQueryTableRsp *rsp = (SQueryTableRsp *)msg;
if (rsp->code != TSDB_CODE_SUCCESS) {
SCH_ERR_JRET(schProcessOnTaskFailure(job, task, rsp->code));
} else {
code = schAsyncSendMsg(job, task, TDMT_VND_RES_READY);
code = schBuildAndSendMsg(job, task, TDMT_VND_RES_READY);
if (code) {
goto _task_error;
}
}
break;
}
case TDMT_VND_RES_READY: {
case TDMT_VND_RES_READY_RSP: {
SResReadyRsp *rsp = (SResReadyRsp *)msg;
if (rsp->code != TSDB_CODE_SUCCESS) {
@ -621,7 +497,7 @@ int32_t schHandleRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg
}
break;
}
case TDMT_VND_FETCH: {
case TDMT_VND_FETCH_RSP: {
SCH_ERR_JRET(rspCode);
SRetrieveTableRsp *rsp = (SRetrieveTableRsp *)msg;
@ -631,6 +507,9 @@ int32_t schHandleRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg
SCH_ERR_JRET(schProcessOnDataFetched(job));
break;
}
case TDMT_VND_DROP_TASK: {
}
default:
qError("unknown msg type:%d received", msgType);
return TSDB_CODE_QRY_INVALID_INPUT;
@ -648,6 +527,211 @@ _return:
}
int32_t schHandleCallback(void* param, const SDataBuf* pMsg, int32_t msgType, int32_t rspCode) {
int32_t code = 0;
SSchCallbackParam *pParam = (SSchCallbackParam *)param;
SSchJob **job = taosHashGet(schMgmt.jobs, &pParam->queryId, sizeof(pParam->queryId));
if (NULL == job || NULL == (*job)) {
qError("taosHashGet queryId:%"PRIx64" not exist", pParam->queryId);
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
SSchTask **task = taosHashGet((*job)->execTasks, &pParam->taskId, sizeof(pParam->taskId));
if (NULL == task || NULL == (*task)) {
qError("taosHashGet taskId:%"PRIx64" not exist", pParam->taskId);
SCH_ERR_JRET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
schProcessRspMsg(*job, *task, msgType, pMsg->pData, pMsg->len, rspCode);
_return:
tfree(param);
SCH_RET(code);
}
int32_t schHandleSubmitCallback(void* param, const SDataBuf* pMsg, int32_t code) {
return schHandleCallback(param, pMsg, TDMT_VND_SUBMIT_RSP, code);
}
int32_t schHandleQueryCallback(void* param, const SDataBuf* pMsg, int32_t code) {
return schHandleCallback(param, pMsg, TDMT_VND_QUERY_RSP, code);
}
int32_t schHandleFetchCallback(void* param, const SDataBuf* pMsg, int32_t code) {
return schHandleCallback(param, pMsg, TDMT_VND_FETCH_RSP, code);
}
int32_t schHandleReadyCallback(void* param, const SDataBuf* pMsg, int32_t code) {
return schHandleCallback(param, pMsg, TDMT_VND_RES_READY_RSP, code);
}
int32_t schHandleDropCallback(void* param, const SDataBuf* pMsg, int32_t code) {
SSchCallbackParam *pParam = (SSchCallbackParam *)param;
qDebug("drop task rsp received, queryId:%"PRIx64 ",taksId:%"PRIx64 ",code:%d", pParam->queryId, pParam->taskId, code);
}
int32_t schGetCallbackFp(int32_t msgType, __async_send_cb_fn_t *fp) {
switch (msgType) {
case TDMT_VND_SUBMIT:
*fp = schHandleSubmitCallback;
break;
case TDMT_VND_QUERY:
*fp = schHandleQueryCallback;
break;
case TDMT_VND_RES_READY:
*fp = schHandleReadyCallback;
break;
case TDMT_VND_FETCH:
*fp = schHandleFetchCallback;
break;
case TDMT_VND_DROP_TASK:
*fp = schHandleDropCallback;
break;
default:
qError("unknown msg type:%d", msgType);
SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
return TSDB_CODE_SUCCESS;
}
int32_t schAsyncSendMsg(void *transport, SEpSet* epSet, uint64_t qId, uint64_t tId, int32_t msgType, void *msg, uint32_t msgSize) {
int32_t code = 0;
SMsgSendInfo* pMsgSendInfo = calloc(1, sizeof(SMsgSendInfo));
if (NULL == pMsgSendInfo) {
qError("calloc %d failed", (int32_t)sizeof(SMsgSendInfo));
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
SSchCallbackParam *param = calloc(1, sizeof(SSchCallbackParam));
if (NULL == param) {
qError("calloc %d failed", (int32_t)sizeof(SSchCallbackParam));
SCH_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
__async_send_cb_fn_t fp = NULL;
SCH_ERR_JRET(schGetCallbackFp(msgType, &fp));
param->queryId = qId;
param->taskId = tId;
pMsgSendInfo->param = param;
pMsgSendInfo->msgInfo.pData = msg;
pMsgSendInfo->msgInfo.len = msgSize;
pMsgSendInfo->msgType = msgType;
pMsgSendInfo->fp = fp;
int64_t transporterId = 0;
SCH_ERR_JRET(asyncSendMsgToServer(transport, epSet, &transporterId, pMsgSendInfo));
return TSDB_CODE_SUCCESS;
_return:
tfree(param);
tfree(pMsgSendInfo);
SCH_RET(code);
}
int32_t schBuildAndSendMsg(SSchJob *job, SSchTask *task, int32_t msgType) {
uint32_t msgSize = 0;
void *msg = NULL;
int32_t code = 0;
switch (msgType) {
case TDMT_VND_SUBMIT: {
if (NULL == task->msg || task->msgLen <= 0) {
qError("submit msg is NULL");
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
msgSize = task->msgLen;
msg = task->msg;
break;
}
case TDMT_VND_QUERY: {
if (NULL == task->msg) {
qError("query msg is NULL");
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
}
msgSize = sizeof(SSubQueryMsg) + task->msgLen;
msg = calloc(1, msgSize);
if (NULL == msg) {
qError("calloc %d failed", msgSize);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
SSubQueryMsg *pMsg = msg;
pMsg->sId = htobe64(schMgmt.sId);
pMsg->queryId = htobe64(job->queryId);
pMsg->taskId = htobe64(task->taskId);
pMsg->contentLen = htonl(task->msgLen);
memcpy(pMsg->msg, task->msg, task->msgLen);
break;
}
case TDMT_VND_RES_READY: {
msgSize = sizeof(SResReadyMsg);
msg = calloc(1, msgSize);
if (NULL == msg) {
qError("calloc %d failed", msgSize);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
SResReadyMsg *pMsg = msg;
pMsg->sId = htobe64(schMgmt.sId);
pMsg->queryId = htobe64(job->queryId);
pMsg->taskId = htobe64(task->taskId);
break;
}
case TDMT_VND_FETCH: {
if (NULL == task) {
SCH_ERR_RET(TSDB_CODE_QRY_APP_ERROR);
}
msgSize = sizeof(SResFetchMsg);
msg = calloc(1, msgSize);
if (NULL == msg) {
qError("calloc %d failed", msgSize);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
SResFetchMsg *pMsg = msg;
pMsg->sId = htobe64(schMgmt.sId);
pMsg->queryId = htobe64(job->queryId);
pMsg->taskId = htobe64(task->taskId);
break;
}
case TDMT_VND_DROP_TASK:{
msgSize = sizeof(STaskDropMsg);
msg = calloc(1, msgSize);
if (NULL == msg) {
qError("calloc %d failed", msgSize);
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
STaskDropMsg *pMsg = msg;
pMsg->sId = htobe64(schMgmt.sId);
pMsg->queryId = htobe64(job->queryId);
pMsg->taskId = htobe64(task->taskId);
break;
}
default:
qError("unknown msg type:%d", msgType);
SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
break;
}
SCH_ERR_JRET(schAsyncSendMsg(job->transport, &task->plan->execEpSet, job->queryId, task->taskId, msgType, msg, msgSize));
return TSDB_CODE_SUCCESS;
_return:
tfree(msg);
SCH_RET(code);
}
int32_t schLaunchTask(SSchJob *job, SSchTask *task) {
@ -664,7 +748,7 @@ int32_t schLaunchTask(SSchJob *job, SSchTask *task) {
int32_t msgType = (plan->type == QUERY_TYPE_MODIFY) ? TDMT_VND_SUBMIT : TDMT_VND_QUERY;
SCH_ERR_RET(schAsyncSendMsg(job, task, msgType));
SCH_ERR_RET(schBuildAndSendMsg(job, task, msgType));
SCH_ERR_RET(schPushTaskToExecList(job, task));
@ -673,6 +757,8 @@ int32_t schLaunchTask(SSchJob *job, SSchTask *task) {
return TSDB_CODE_SUCCESS;
}
int32_t schLaunchJob(SSchJob *job) {
SSchLevel *level = taosArrayGet(job->levels, job->levelIdx);
for (int32_t i = 0; i < level->taskNum; ++i) {
@ -690,7 +776,7 @@ void schDropJobAllTasks(SSchJob *job) {
while (pIter) {
SSchTask *task = *(SSchTask **)pIter;
schAsyncSendMsg(job, task, TDMT_VND_DROP_TASK);
schBuildAndSendMsg(job, task, TDMT_VND_DROP_TASK);
pIter = taosHashIterate(job->succTasks, pIter);
}
@ -699,7 +785,7 @@ void schDropJobAllTasks(SSchJob *job) {
while (pIter) {
SSchTask *task = *(SSchTask **)pIter;
schAsyncSendMsg(job, task, TDMT_VND_DROP_TASK);
schBuildAndSendMsg(job, task, TDMT_VND_DROP_TASK);
pIter = taosHashIterate(job->succTasks, pIter);
}
@ -717,7 +803,7 @@ int32_t schedulerInit(SSchedulerCfg *cfg) {
SCH_ERR_LRET(TSDB_CODE_QRY_OUT_OF_MEMORY, "init %d schduler jobs failed", schMgmt.cfg.maxJobNum);
}
schMgmt.schedulerId = 1; //TODO GENERATE A UUID
schMgmt.sId = 1; //TODO GENERATE A UUID
return TSDB_CODE_SUCCESS;
}

14
source/libs/scheduler/test/schedulerTests.cpp
View File

@ -36,7 +36,7 @@
namespace {
extern "C" int32_t schHandleRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode);
extern "C" int32_t schProcessRspMsg(SSchJob *job, SSchTask *task, int32_t msgType, char *msg, int32_t msgSize, int32_t rspCode);
void schtBuildQueryDag(SQueryDag *dag) {
uint64_t qId = 0x0000000000000001;
@ -182,7 +182,7 @@ void *schtSendRsp(void *param) {
SShellSubmitRspMsg rsp = {0};
rsp.affectedRows = 10;
schHandleRspMsg(job, task, TDMT_VND_SUBMIT, (char *)&rsp, sizeof(rsp), 0);
schProcessRspMsg(job, task, TDMT_VND_SUBMIT, (char *)&rsp, sizeof(rsp), 0);
pIter = taosHashIterate(job->execTasks, pIter);
}
@ -227,7 +227,7 @@ TEST(queryTest, normalCase) {
SSchTask *task = *(SSchTask **)pIter;
SQueryTableRsp rsp = {0};
code = schHandleRspMsg(job, task, TDMT_VND_QUERY, (char *)&rsp, sizeof(rsp), 0);
code = schProcessRspMsg(job, task, TDMT_VND_QUERY, (char *)&rsp, sizeof(rsp), 0);
ASSERT_EQ(code, 0);
pIter = taosHashIterate(job->execTasks, pIter);
@ -238,7 +238,7 @@ TEST(queryTest, normalCase) {
SSchTask *task = *(SSchTask **)pIter;
SResReadyRsp rsp = {0};
code = schHandleRspMsg(job, task, TDMT_VND_RES_READY, (char *)&rsp, sizeof(rsp), 0);
code = schProcessRspMsg(job, task, TDMT_VND_RES_READY, (char *)&rsp, sizeof(rsp), 0);
ASSERT_EQ(code, 0);
pIter = taosHashIterate(job->execTasks, pIter);
@ -249,7 +249,7 @@ TEST(queryTest, normalCase) {
SSchTask *task = *(SSchTask **)pIter;
SQueryTableRsp rsp = {0};
code = schHandleRspMsg(job, task, TDMT_VND_QUERY, (char *)&rsp, sizeof(rsp), 0);
code = schProcessRspMsg(job, task, TDMT_VND_QUERY, (char *)&rsp, sizeof(rsp), 0);
ASSERT_EQ(code, 0);
pIter = taosHashIterate(job->execTasks, pIter);
@ -260,7 +260,7 @@ TEST(queryTest, normalCase) {
SSchTask *task = *(SSchTask **)pIter;
SResReadyRsp rsp = {0};
code = schHandleRspMsg(job, task, TDMT_VND_RES_READY, (char *)&rsp, sizeof(rsp), 0);
code = schProcessRspMsg(job, task, TDMT_VND_RES_READY, (char *)&rsp, sizeof(rsp), 0);
ASSERT_EQ(code, 0);
pIter = taosHashIterate(job->execTasks, pIter);
@ -269,7 +269,7 @@ TEST(queryTest, normalCase) {
SRetrieveTableRsp rsp = {0};
rsp.completed = 1;
rsp.numOfRows = 10;
code = schHandleRspMsg(job, NULL, TDMT_VND_FETCH, (char *)&rsp, sizeof(rsp), 0);
code = schProcessRspMsg(job, NULL, TDMT_VND_FETCH, (char *)&rsp, sizeof(rsp), 0);
ASSERT_EQ(code, 0);

1
source/util/src/terror.c
View File

@ -79,6 +79,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FILE_CORRUPTED, "Data file corrupted")
TAOS_DEFINE_ERROR(TSDB_CODE_CHECKSUM_ERROR, "Checksum error")
TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_MSG, "Invalid config message")
TAOS_DEFINE_ERROR(TSDB_CODE_MSG_NOT_PROCESSED, "Message not processed")
TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_PARA, "Invalid parameters")
TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory")
TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs")
TAOS_DEFINE_ERROR(TSDB_CODE_REF_ID_REMOVED, "Ref ID is removed")

6
tests/script/general/user/testSuite.sim
View File

@ -1,6 +0,0 @@
run general/user/basic1.sim
run general/user/monitor.sim
run general/user/pass_alter.sim
run general/user/pass_len.sim
run general/user/user_create.sim
run general/user/user_len.sim

3
tests/script/jenkins/basic.txt
View File

@ -10,4 +10,7 @@
# ---- table
./test.sh -f general/table/basic1.sim
# ---- dnode
./test.sh -f unique/dnode/basic1.sim
#======================b1-end===============

231
tests/script/jenkins/simple.txt
View File

@ -1,231 +0,0 @@
cd ../../../debug; cmake ..
cd ../../../debug; make
#./test.sh -f general/alter/cached_schema_after_alter.sim
./test.sh -f general/alter/count.sim
./test.sh -f general/alter/import.sim
#./test.sh -f general/alter/insert1.sim
./test.sh -f general/alter/insert2.sim
./test.sh -f general/alter/metrics.sim
./test.sh -f general/alter/table.sim
./test.sh -f general/cache/new_metrics.sim
./test.sh -f general/cache/restart_metrics.sim
./test.sh -f general/cache/restart_table.sim
./test.sh -f general/connection/connection.sim
./test.sh -f general/column/commit.sim
./test.sh -f general/column/metrics.sim
./test.sh -f general/column/table.sim
./test.sh -f general/compress/commitlog.sim
./test.sh -f general/compress/compress.sim
./test.sh -f general/compress/compress2.sim
./test.sh -f general/compress/uncompress.sim
./test.sh -f general/compute/avg.sim
./test.sh -f general/compute/bottom.sim
./test.sh -f general/compute/count.sim
./test.sh -f general/compute/diff.sim
./test.sh -f general/compute/diff2.sim
./test.sh -f general/compute/first.sim
./test.sh -f general/compute/interval.sim
./test.sh -f general/compute/last.sim
./test.sh -f general/compute/leastsquare.sim
./test.sh -f general/compute/max.sim
./test.sh -f general/compute/min.sim
./test.sh -f general/compute/null.sim
./test.sh -f general/compute/percentile.sim
./test.sh -f general/compute/stddev.sim
./test.sh -f general/compute/sum.sim
./test.sh -f general/compute/top.sim
./test.sh -f general/db/alter_option.sim
./test.sh -f general/db/alter_tables_d2.sim
./test.sh -f general/db/alter_tables_v1.sim
./test.sh -f general/db/alter_tables_v4.sim
./test.sh -f general/db/alter_vgroups.sim
./test.sh -f general/db/basic.sim
./test.sh -f general/db/basic1.sim
./test.sh -f general/db/basic2.sim
./test.sh -f general/db/basic3.sim
./test.sh -f general/db/basic4.sim
./test.sh -f general/db/basic5.sim
./test.sh -f general/db/delete_reuse1.sim
./test.sh -f general/db/delete_reuse2.sim
./test.sh -f general/db/delete_reusevnode.sim
./test.sh -f general/db/delete_reusevnode2.sim
./test.sh -f general/db/delete_writing1.sim
./test.sh -f general/db/delete_writing2.sim
./test.sh -f general/db/delete.sim
./test.sh -f general/db/len.sim
./test.sh -f general/db/repeat.sim
./test.sh -f general/db/tables.sim
./test.sh -f general/db/vnodes.sim
./test.sh -f general/field/2.sim
./test.sh -f general/field/3.sim
./test.sh -f general/field/4.sim
./test.sh -f general/field/5.sim
./test.sh -f general/field/6.sim
./test.sh -f general/field/bigint.sim
./test.sh -f general/field/binary.sim
./test.sh -f general/field/bool.sim
./test.sh -f general/field/single.sim
./test.sh -f general/field/smallint.sim
./test.sh -f general/field/tinyint.sim
./test.sh -f general/http/restful.sim
./test.sh -f general/http/restful_insert.sim
./test.sh -f general/http/restful_limit.sim
./test.sh -f general/http/restful_full.sim
./test.sh -f general/http/prepare.sim
./test.sh -f general/http/telegraf.sim
./test.sh -f general/http/grafana_bug.sim
./test.sh -f general/http/grafana.sim
./test.sh -f general/import/basic.sim
./test.sh -f general/import/commit.sim
./test.sh -f general/import/large.sim
./test.sh -f general/import/replica1.sim
./test.sh -f general/insert/basic.sim
./test.sh -f general/insert/insert_drop.sim
./test.sh -f general/insert/query_block1_memory.sim
./test.sh -f general/insert/query_block2_memory.sim
./test.sh -f general/insert/query_block1_file.sim
./test.sh -f general/insert/query_block2_file.sim
./test.sh -f general/insert/query_file_memory.sim
./test.sh -f general/insert/query_multi_file.sim
./test.sh -f general/insert/tcp.sim
./test.sh -f general/parser/alter.sim
./test.sh -f general/parser/alter1.sim
./test.sh -f general/parser/alter_stable.sim
./test.sh -f general/parser/auto_create_tb.sim
./test.sh -f general/parser/auto_create_tb_drop_tb.sim
./test.sh -f general/parser/col_arithmetic_operation.sim
./test.sh -f general/parser/columnValue.sim
./test.sh -f general/parser/commit.sim
./test.sh -f general/parser/create_db.sim
./test.sh -f general/parser/create_mt.sim
./test.sh -f general/parser/create_tb.sim
./test.sh -f general/parser/dbtbnameValidate.sim
./test.sh -f general/parser/import_commit1.sim
./test.sh -f general/parser/import_commit2.sim
./test.sh -f general/parser/import_commit3.sim
./test.sh -f general/parser/insert_tb.sim
./test.sh -f general/parser/first_last.sim
#./test.sh -f general/parser/import_file.sim
./test.sh -f general/parser/lastrow.sim
./test.sh -f general/parser/nchar.sim
./test.sh -f general/parser/null_char.sim
./test.sh -f general/parser/single_row_in_tb.sim
./test.sh -f general/parser/select_from_cache_disk.sim
./test.sh -f general/parser/mixed_blocks.sim
./test.sh -f general/parser/selectResNum.sim
./test.sh -f general/parser/limit.sim
./test.sh -f general/parser/limit1.sim
./test.sh -f general/parser/limit1_tblocks100.sim
./test.sh -f general/parser/select_across_vnodes.sim
./test.sh -f general/parser/slimit1.sim
./test.sh -f general/parser/tbnameIn.sim
./test.sh -f general/parser/projection_limit_offset.sim
./test.sh -f general/parser/limit2.sim
./test.sh -f general/parser/fill.sim
./test.sh -f general/parser/fill_stb.sim
./test.sh -f general/parser/where.sim
./test.sh -f general/parser/slimit.sim
./test.sh -f general/parser/select_with_tags.sim
./test.sh -f general/parser/interp.sim
./test.sh -f general/parser/tags_dynamically_specifiy.sim
./test.sh -f general/parser/groupby.sim
./test.sh -f general/parser/set_tag_vals.sim
#./test.sh -f general/parser/slimit_alter_tags.sim
./test.sh -f general/parser/join.sim
./test.sh -f general/parser/join_multivnode.sim
./test.sh -f general/parser/binary_escapeCharacter.sim
./test.sh -f general/parser/bug.sim
./test.sh -f general/parser/repeatAlter.sim
./test.sh -f general/stable/disk.sim
./test.sh -f general/stable/dnode3.sim
./test.sh -f general/stable/metrics.sim
./test.sh -f general/stable/refcount.sim
./test.sh -f general/stable/show.sim
./test.sh -f general/stable/values.sim
./test.sh -f general/stable/vnode3.sim
./test.sh -f general/table/autocreate.sim
./test.sh -f general/table/basic1.sim
./test.sh -f general/table/basic2.sim
./test.sh -f general/table/basic3.sim
./test.sh -f general/table/bigint.sim
./test.sh -f general/table/binary.sim
./test.sh -f general/table/bool.sim
./test.sh -f general/table/column_name.sim
./test.sh -f general/table/column_num.sim
./test.sh -f general/table/column_value.sim
./test.sh -f general/table/column2.sim
./test.sh -f general/table/date.sim
./test.sh -f general/table/db.table.sim
./test.sh -f general/table/delete_reuse1.sim
./test.sh -f general/table/delete_reuse2.sim
./test.sh -f general/table/delete_writing.sim
./test.sh -f general/table/describe.sim
./test.sh -f general/table/double.sim
./test.sh -f general/table/fill.sim
./test.sh -f general/table/float.sim
./test.sh -f general/table/int.sim
./test.sh -f general/table/limit.sim
./test.sh -f general/table/smallint.sim
./test.sh -f general/table/table_len.sim
./test.sh -f general/table/table.sim
./test.sh -f general/table/tinyint.sim
./test.sh -f general/table/vgroup.sim
./test.sh -f general/tag/3.sim
./test.sh -f general/tag/4.sim
./test.sh -f general/tag/5.sim
./test.sh -f general/tag/6.sim
./test.sh -f general/tag/add.sim
./test.sh -f general/tag/bigint.sim
./test.sh -f general/tag/binary_binary.sim
./test.sh -f general/tag/binary.sim
./test.sh -f general/tag/bool_binary.sim
./test.sh -f general/tag/bool_int.sim
./test.sh -f general/tag/bool.sim
./test.sh -f general/tag/change.sim
./test.sh -f general/tag/column.sim
#./test.sh -f general/tag/commit.sim
./test.sh -f general/tag/create.sim
./test.sh -f general/tag/delete.sim
./test.sh -f general/tag/double.sim
./test.sh -f general/tag/filter.sim
./test.sh -f general/tag/float.sim
./test.sh -f general/tag/int_binary.sim
./test.sh -f general/tag/int_float.sim
./test.sh -f general/tag/int.sim
./test.sh -f general/tag/set.sim
./test.sh -f general/tag/smallint.sim
./test.sh -f general/tag/tinyint.sim
./test.sh -f general/user/authority.sim
./test.sh -f general/user/monitor.sim
./test.sh -f general/user/pass_alter.sim
./test.sh -f general/user/pass_len.sim
./test.sh -f general/user/user_create.sim
./test.sh -f general/user/user_len.sim
./test.sh -f general/vector/metrics_field.sim
./test.sh -f general/vector/metrics_mix.sim
./test.sh -f general/vector/metrics_query.sim
./test.sh -f general/vector/metrics_tag.sim
./test.sh -f general/vector/metrics_time.sim
./test.sh -f general/vector/multi.sim
./test.sh -f general/vector/single.sim
./test.sh -f general/vector/table_field.sim
./test.sh -f general/vector/table_mix.sim
./test.sh -f general/vector/table_query.sim
./test.sh -f general/vector/table_time.sim

127
tests/script/jenkins/unique.txt
View File

@ -1,127 +0,0 @@
cd ../../../debug; cmake ..
cd ../../../debug; make
./test.sh -f unique/big/balance.sim
./test.sh -f unique/big/maxvnodes.sim
./test.sh -f unique/big/tcp.sim
./test.sh -f unique/cluster/alter.sim
./test.sh -f unique/cluster/balance1.sim
./test.sh -f unique/cluster/balance2.sim
./test.sh -f unique/cluster/balance3.sim
./test.sh -f unique/cluster/cache.sim
./test.sh -f unique/cluster/vgroup100.sim
./test.sh -f unique/column/replica3.sim
./test.sh -f unique/db/commit.sim
./test.sh -f unique/db/delete.sim
./test.sh -f unique/db/delete_part.sim
./test.sh -f unique/db/replica_add12.sim
./test.sh -f unique/db/replica_add13.sim
./test.sh -f unique/db/replica_add23.sim
./test.sh -f unique/db/replica_reduce21.sim
./test.sh -f unique/db/replica_reduce32.sim
./test.sh -f unique/db/replica_reduce31.sim
./test.sh -f unique/db/replica_part.sim
./test.sh -f unique/dnode/alternativeRole.sim
./test.sh -f unique/dnode/monitor.sim
./test.sh -f unique/dnode/monitor_bug.sim
./test.sh -f unique/dnode/simple.sim
./test.sh -f unique/dnode/balance1.sim
./test.sh -f unique/dnode/balance2.sim
./test.sh -f unique/dnode/balance3.sim
./test.sh -f unique/dnode/balancex.sim
./test.sh -f unique/dnode/data1.sim
./test.sh -f unique/dnode/m2.sim
./test.sh -f unique/dnode/m3.sim
./test.sh -f unique/dnode/lossdata.sim
./test.sh -f unique/dnode/offline1.sim
./test.sh -f unique/dnode/offline2.sim
./test.sh -f unique/dnode/offline3.sim
./test.sh -f unique/dnode/reason.sim
./test.sh -f unique/dnode/remove1.sim
./test.sh -f unique/dnode/remove2.sim
./test.sh -f unique/dnode/vnode_clean.sim
./test.sh -f unique/http/admin.sim
./test.sh -f unique/http/opentsdb.sim
./test.sh -f unique/import/replica2.sim
./test.sh -f unique/import/replica3.sim
./test.sh -f unique/stable/balance_replica1.sim
./test.sh -f unique/stable/dnode2_stop.sim
./test.sh -f unique/stable/dnode2.sim
./test.sh -f unique/stable/dnode3.sim
./test.sh -f unique/stable/replica2_dnode4.sim
./test.sh -f unique/stable/replica2_vnode3.sim
./test.sh -f unique/stable/replica3_dnode6.sim
./test.sh -f unique/stable/replica3_vnode3.sim
./test.sh -f unique/mnode/mgmt20.sim
./test.sh -f unique/mnode/mgmt21.sim
./test.sh -f unique/mnode/mgmt22.sim
./test.sh -f unique/mnode/mgmt23.sim
./test.sh -f unique/mnode/mgmt24.sim
./test.sh -f unique/mnode/mgmt25.sim
./test.sh -f unique/mnode/mgmt26.sim
./test.sh -f unique/mnode/mgmt30.sim
./test.sh -f unique/mnode/mgmt33.sim
./test.sh -f unique/mnode/mgmt34.sim
./test.sh -f unique/mnode/mgmtr2.sim
./test.sh -f unique/vnode/many.sim
./test.sh -f unique/vnode/replica2_basic2.sim
./test.sh -f unique/vnode/replica2_repeat.sim
./test.sh -f unique/vnode/replica3_basic.sim
./test.sh -f unique/vnode/replica3_repeat.sim
./test.sh -f unique/vnode/replica3_vgroup.sim
./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim
#./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim
./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim
./test.sh -f unique/arbitrator/dn3_mn1_multiCreateDropTable.sim
#./test.sh -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim
#./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim
./test.sh -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim
./test.sh -f unique/arbitrator/dn3_mn1_replica_change.sim
#./test.sh -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim
# lower the priority while file corruption
#./test.sh -f unique/arbitrator/dn3_mn1_vnode_change.sim
#./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim
#./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim
#./test.sh -f unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim
./test.sh -f unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim
./test.sh -f unique/arbitrator/dn3_mn1_vnode_delDir.sim
./test.sh -f unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim
./test.sh -f unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim
./test.sh -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim
./test.sh -f unique/arbitrator/dn3_mn2_killDnode.sim
./test.sh -f unique/arbitrator/insert_duplicationTs.sim
./test.sh -f unique/arbitrator/offline_replica2_alterTable_online.sim
./test.sh -f unique/arbitrator/offline_replica2_alterTag_online.sim
./test.sh -f unique/arbitrator/offline_replica2_createTable_online.sim
./test.sh -f unique/arbitrator/offline_replica2_dropDb_online.sim
./test.sh -f unique/arbitrator/offline_replica2_dropTable_online.sim
./test.sh -f unique/arbitrator/offline_replica3_alterTable_online.sim
./test.sh -f unique/arbitrator/offline_replica3_alterTag_online.sim
./test.sh -f unique/arbitrator/offline_replica3_createTable_online.sim
./test.sh -f unique/arbitrator/offline_replica3_dropDb_online.sim
./test.sh -f unique/arbitrator/offline_replica3_dropTable_online.sim
./test.sh -f unique/arbitrator/replica_changeWithArbitrator.sim
./test.sh -f unique/arbitrator/sync_replica2_alterTable_add.sim
./test.sh -f unique/arbitrator/sync_replica2_alterTable_drop.sim
./test.sh -f unique/arbitrator/sync_replica2_dropDb.sim
./test.sh -f unique/arbitrator/sync_replica2_dropTable.sim
./test.sh -f unique/arbitrator/sync_replica3_alterTable_add.sim
./test.sh -f unique/arbitrator/sync_replica3_alterTable_drop.sim
./test.sh -f unique/arbitrator/sync_replica3_dropDb.sim
./test.sh -f unique/arbitrator/sync_replica3_dropTable.sim
./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim
./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim
./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim
./test.sh -f unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim

393
tests/script/jenkins/wbasic.txt
View File

@ -1,393 +0,0 @@
wtest.bat -f issue/TD-2677.sim
wtest.bat -f issue/TD-2680.sim
wtest.bat -f issue/TD-2713.sim
wtest.bat -f general/alter/cached_schema_after_alter.sim
wtest.bat -f general/alter/count.sim
wtest.bat -f general/alter/dnode.sim
wtest.bat -f general/alter/import.sim
wtest.bat -f general/alter/insert1.sim
wtest.bat -f general/alter/insert2.sim
wtest.bat -f general/alter/metrics.sim
wtest.bat -f general/alter/table.sim
wtest.bat -f general/cache/new_metrics.sim
wtest.bat -f general/cache/restart_metrics.sim
wtest.bat -f general/cache/restart_table.sim
wtest.bat -f general/connection/connection.sim
wtest.bat -f general/column/commit.sim
wtest.bat -f general/column/metrics.sim
wtest.bat -f general/column/table.sim
wtest.bat -f general/compress/commitlog.sim
wtest.bat -f general/compress/compress.sim
wtest.bat -f general/compress/compress2.sim
wtest.bat -f general/compress/uncompress.sim
wtest.bat -f general/compute/avg.sim
wtest.bat -f general/compute/bottom.sim
wtest.bat -f general/compute/count.sim
wtest.bat -f general/compute/diff.sim
wtest.bat -f general/compute/diff2.sim
wtest.bat -f general/compute/first.sim
wtest.bat -f general/compute/interval.sim
wtest.bat -f general/compute/last.sim
wtest.bat -f general/compute/leastsquare.sim
wtest.bat -f general/compute/max.sim
wtest.bat -f general/compute/min.sim
wtest.bat -f general/compute/null.sim
wtest.bat -f general/compute/percentile.sim
wtest.bat -f general/compute/stddev.sim
wtest.bat -f general/compute/sum.sim
wtest.bat -f general/compute/top.sim
wtest.bat -f general/db/alter_option.sim
wtest.bat -f general/db/alter_tables_d2.sim
wtest.bat -f general/db/alter_tables_v1.sim
wtest.bat -f general/db/alter_tables_v4.sim
wtest.bat -f general/db/alter_vgroups.sim
wtest.bat -f general/db/basic.sim
wtest.bat -f general/db/basic1.sim
wtest.bat -f general/db/basic2.sim
wtest.bat -f general/db/basic3.sim
wtest.bat -f general/db/basic4.sim
wtest.bat -f general/db/basic5.sim
wtest.bat -f general/db/delete_reuse1.sim
wtest.bat -f general/db/delete_reuse2.sim
wtest.bat -f general/db/delete_reusevnode.sim
wtest.bat -f general/db/delete_reusevnode2.sim
wtest.bat -f general/db/delete_writing1.sim
wtest.bat -f general/db/delete_writing2.sim
wtest.bat -f general/db/delete.sim
wtest.bat -f general/db/len.sim
wtest.bat -f general/db/repeat.sim
wtest.bat -f general/db/tables.sim
wtest.bat -f general/db/vnodes.sim
wtest.bat -f general/db/nosuchfile.sim
wtest.bat -f general/field/2.sim
wtest.bat -f general/field/3.sim
wtest.bat -f general/field/4.sim
wtest.bat -f general/field/5.sim
wtest.bat -f general/field/6.sim
wtest.bat -f general/field/bigint.sim
wtest.bat -f general/field/binary.sim
wtest.bat -f general/field/bool.sim
wtest.bat -f general/field/single.sim
wtest.bat -f general/field/smallint.sim
wtest.bat -f general/field/tinyint.sim
wtest.bat -f general/http/autocreate.sim
wtest.bat -f general/http/chunked.sim
wtest.bat -f general/http/gzip.sim
wtest.bat -f general/http/restful.sim
wtest.bat -f general/http/restful_insert.sim
wtest.bat -f general/http/restful_limit.sim
wtest.bat -f general/http/restful_full.sim
wtest.bat -f general/http/prepare.sim
wtest.bat -f general/http/telegraf.sim
wtest.bat -f general/http/grafana_bug.sim
wtest.bat -f general/http/grafana.sim
wtest.bat -f general/import/basic.sim
wtest.bat -f general/import/commit.sim
wtest.bat -f general/import/large.sim
wtest.bat -f general/import/replica1.sim
wtest.bat -f general/insert/basic.sim
wtest.bat -f general/insert/insert_drop.sim
wtest.bat -f general/insert/query_block1_memory.sim
wtest.bat -f general/insert/query_block2_memory.sim
wtest.bat -f general/insert/query_block1_file.sim
wtest.bat -f general/insert/query_block2_file.sim
wtest.bat -f general/insert/query_file_memory.sim
wtest.bat -f general/insert/query_multi_file.sim
wtest.bat -f general/insert/tcp.sim
wtest.bat -f general/parser/alter.sim
wtest.bat -f general/parser/alter1.sim
wtest.bat -f general/parser/alter_stable.sim
wtest.bat -f general/parser/auto_create_tb.sim
wtest.bat -f general/parser/auto_create_tb_drop_tb.sim
wtest.bat -f general/parser/col_arithmetic_operation.sim
wtest.bat -f general/parser/columnValue.sim
wtest.bat -f general/parser/commit.sim
wtest.bat -f general/parser/create_db.sim
wtest.bat -f general/parser/create_mt.sim
wtest.bat -f general/parser/create_tb.sim
wtest.bat -f general/parser/dbtbnameValidate.sim
wtest.bat -f general/parser/import_commit1.sim
wtest.bat -f general/parser/import_commit2.sim
wtest.bat -f general/parser/import_commit3.sim
wtest.bat -f general/parser/insert_tb.sim
wtest.bat -f general/parser/first_last.sim
wtest.bat -f general/parser/lastrow.sim
wtest.bat -f general/parser/nchar.sim
wtest.bat -f general/parser/null_char.sim
wtest.bat -f general/parser/single_row_in_tb.sim
wtest.bat -f general/parser/select_from_cache_disk.sim
wtest.bat -f general/parser/mixed_blocks.sim
wtest.bat -f general/parser/selectResNum.sim
wtest.bat -f general/parser/limit.sim
wtest.bat -f general/parser/limit1.sim
wtest.bat -f general/parser/limit1_tblocks100.sim
wtest.bat -f general/parser/select_across_vnodes.sim
wtest.bat -f general/parser/slimit1.sim
wtest.bat -f general/parser/tbnameIn.sim
wtest.bat -f general/parser/projection_limit_offset.sim
wtest.bat -f general/parser/limit2.sim
wtest.bat -f general/parser/fill.sim
wtest.bat -f general/parser/fill_stb.sim
wtest.bat -f general/parser/where.sim
wtest.bat -f general/parser/slimit.sim
wtest.bat -f general/parser/select_with_tags.sim
wtest.bat -f general/parser/interp.sim
wtest.bat -f general/parser/tags_dynamically_specifiy.sim
wtest.bat -f general/parser/groupby.sim
wtest.bat -f general/parser/set_tag_vals.sim
wtest.bat -f general/parser/tags_filter.sim
wtest.bat -f general/parser/slimit_alter_tags.sim
wtest.bat -f general/parser/join.sim
wtest.bat -f general/parser/join_multivnode.sim
wtest.bat -f general/parser/binary_escapeCharacter.sim
wtest.bat -f general/parser/repeatAlter.sim
wtest.bat -f general/parser/union.sim
wtest.bat -f general/parser/topbot.sim
wtest.bat -f general/parser/function.sim
wtest.bat -f general/stable/disk.sim
wtest.bat -f general/stable/dnode3.sim
wtest.bat -f general/stable/metrics.sim
wtest.bat -f general/stable/refcount.sim
wtest.bat -f general/stable/show.sim
wtest.bat -f general/stable/values.sim
wtest.bat -f general/stable/vnode3.sim
wtest.bat -f general/table/autocreate.sim
wtest.bat -f general/table/basic1.sim
wtest.bat -f general/table/basic2.sim
wtest.bat -f general/table/basic3.sim
wtest.bat -f general/table/bigint.sim
wtest.bat -f general/table/binary.sim
wtest.bat -f general/table/bool.sim
wtest.bat -f general/table/column_name.sim
wtest.bat -f general/table/column_num.sim
wtest.bat -f general/table/column_value.sim
wtest.bat -f general/table/column2.sim
wtest.bat -f general/table/date.sim
wtest.bat -f general/table/db.table.sim
wtest.bat -f general/table/delete_reuse1.sim
wtest.bat -f general/table/delete_reuse2.sim
wtest.bat -f general/table/delete_writing.sim
wtest.bat -f general/table/describe.sim
wtest.bat -f general/table/double.sim
wtest.bat -f general/table/fill.sim
wtest.bat -f general/table/float.sim
wtest.bat -f general/table/int.sim
wtest.bat -f general/table/limit.sim
wtest.bat -f general/table/smallint.sim
wtest.bat -f general/table/table_len.sim
wtest.bat -f general/table/table.sim
wtest.bat -f general/table/tinyint.sim
wtest.bat -f general/table/vgroup.sim
wtest.bat -f general/tag/3.sim
wtest.bat -f general/tag/4.sim
wtest.bat -f general/tag/5.sim
wtest.bat -f general/tag/6.sim
wtest.bat -f general/tag/add.sim
wtest.bat -f general/tag/bigint.sim
wtest.bat -f general/tag/binary_binary.sim
wtest.bat -f general/tag/binary.sim
wtest.bat -f general/tag/bool_binary.sim
wtest.bat -f general/tag/bool_int.sim
wtest.bat -f general/tag/bool.sim
wtest.bat -f general/tag/change.sim
wtest.bat -f general/tag/column.sim
wtest.bat -f general/tag/commit.sim
wtest.bat -f general/tag/create.sim
wtest.bat -f general/tag/delete.sim
wtest.bat -f general/tag/double.sim
wtest.bat -f general/tag/filter.sim
wtest.bat -f general/tag/float.sim
wtest.bat -f general/tag/int_binary.sim
wtest.bat -f general/tag/int_float.sim
wtest.bat -f general/tag/int.sim
wtest.bat -f general/tag/set.sim
wtest.bat -f general/tag/smallint.sim
wtest.bat -f general/tag/tinyint.sim
wtest.bat -f general/user/authority.sim
wtest.bat -f general/user/monitor.sim
wtest.bat -f general/user/pass_alter.sim
wtest.bat -f general/user/pass_len.sim
wtest.bat -f general/user/user_create.sim
wtest.bat -f general/user/user_len.sim
wtest.bat -f general/vector/metrics_field.sim
wtest.bat -f general/vector/metrics_mix.sim
wtest.bat -f general/vector/metrics_query.sim
wtest.bat -f general/vector/metrics_tag.sim
wtest.bat -f general/vector/metrics_time.sim
wtest.bat -f general/vector/multi.sim
wtest.bat -f general/vector/single.sim
wtest.bat -f general/vector/table_field.sim
wtest.bat -f general/vector/table_mix.sim
wtest.bat -f general/vector/table_query.sim
wtest.bat -f general/vector/table_time.sim
wtest.bat -f general/wal/sync.sim
wtest.bat -f general/wal/kill.sim
wtest.bat -f general/wal/maxtables.sim
wtest.bat -f unique/account/account_create.sim
wtest.bat -f unique/account/account_delete.sim
wtest.bat -f unique/account/account_len.sim
wtest.bat -f unique/account/authority.sim
wtest.bat -f unique/account/basic.sim
wtest.bat -f unique/account/paras.sim
wtest.bat -f unique/account/pass_alter.sim
wtest.bat -f unique/account/pass_len.sim
wtest.bat -f unique/account/usage.sim
wtest.bat -f unique/account/user_create.sim
wtest.bat -f unique/account/user_len.sim
wtest.bat -f unique/big/balance.sim
wtest.bat -f unique/big/maxvnodes.sim
wtest.bat -f unique/big/tcp.sim
wtest.bat -f unique/cluster/alter.sim
wtest.bat -f unique/cluster/balance1.sim
wtest.bat -f unique/cluster/balance2.sim
wtest.bat -f unique/cluster/balance3.sim
wtest.bat -f unique/cluster/cache.sim
wtest.bat -f unique/cluster/vgroup100.sim
wtest.bat -f unique/column/replica3.sim
wtest.bat -f unique/db/commit.sim
wtest.bat -f unique/db/delete.sim
wtest.bat -f unique/db/delete_part.sim
wtest.bat -f unique/db/replica_add12.sim
wtest.bat -f unique/db/replica_add13.sim
wtest.bat -f unique/db/replica_add23.sim
wtest.bat -f unique/db/replica_reduce21.sim
wtest.bat -f unique/db/replica_reduce32.sim
wtest.bat -f unique/db/replica_reduce31.sim
wtest.bat -f unique/db/replica_part.sim
wtest.bat -f unique/dnode/alternativeRole.sim
wtest.bat -f unique/dnode/monitor.sim
wtest.bat -f unique/dnode/monitor_bug.sim
wtest.bat -f unique/dnode/simple.sim
wtest.bat -f unique/dnode/balance1.sim
wtest.bat -f unique/dnode/balance2.sim
wtest.bat -f unique/dnode/balance3.sim
wtest.bat -f unique/dnode/balancex.sim
wtest.bat -f unique/dnode/data1.sim
wtest.bat -f unique/dnode/m2.sim
wtest.bat -f unique/dnode/m3.sim
wtest.bat -f unique/dnode/lossdata.sim
wtest.bat -f unique/dnode/offline1.sim
wtest.bat -f unique/dnode/offline2.sim
wtest.bat -f unique/dnode/offline3.sim
wtest.bat -f unique/dnode/reason.sim
wtest.bat -f unique/dnode/remove1.sim
wtest.bat -f unique/dnode/remove2.sim
wtest.bat -f unique/dnode/vnode_clean.sim
wtest.bat -f unique/http/admin.sim
wtest.bat -f unique/http/opentsdb.sim
wtest.bat -f unique/import/replica2.sim
wtest.bat -f unique/import/replica3.sim
wtest.bat -f unique/stable/balance_replica1.sim
wtest.bat -f unique/stable/dnode2_stop.sim
wtest.bat -f unique/stable/dnode2.sim
wtest.bat -f unique/stable/dnode3.sim
wtest.bat -f unique/stable/replica2_dnode4.sim
wtest.bat -f unique/stable/replica2_vnode3.sim
wtest.bat -f unique/stable/replica3_dnode6.sim
wtest.bat -f unique/stable/replica3_vnode3.sim
wtest.bat -f unique/mnode/mgmt20.sim
wtest.bat -f unique/mnode/mgmt21.sim
wtest.bat -f unique/mnode/mgmt22.sim
wtest.bat -f unique/mnode/mgmt23.sim
wtest.bat -f unique/mnode/mgmt24.sim
wtest.bat -f unique/mnode/mgmt25.sim
wtest.bat -f unique/mnode/mgmt26.sim
wtest.bat -f unique/mnode/mgmt30.sim
wtest.bat -f unique/mnode/mgmt33.sim
wtest.bat -f unique/mnode/mgmt34.sim
wtest.bat -f unique/mnode/mgmtr2.sim
wtest.bat -f unique/vnode/many.sim
wtest.bat -f unique/vnode/replica2_basic2.sim
wtest.bat -f unique/vnode/replica2_repeat.sim
wtest.bat -f unique/vnode/replica3_basic.sim
wtest.bat -f unique/vnode/replica3_repeat.sim
wtest.bat -f unique/vnode/replica3_vgroup.sim
wtest.bat -f general/stream/metrics_del.sim
wtest.bat -f general/stream/metrics_replica1_vnoden.sim
wtest.bat -f general/stream/restart_stream.sim
wtest.bat -f general/stream/stream_3.sim
wtest.bat -f general/stream/stream_restart.sim
wtest.bat -f general/stream/table_del.sim
wtest.bat -f general/stream/table_replica1_vnoden.sim
wtest.bat -f unique/arbitrator/check_cluster_cfg_para.sim
#wtest.bat -f unique/arbitrator/dn2_mn1_cache_file_sync.sim
wtest.bat -f unique/arbitrator/dn3_mn1_full_createTableFail.sim
wtest.bat -f unique/arbitrator/dn3_mn1_multiCreateDropTable.sim
#wtest.bat -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim
#wtest.bat -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim
wtest.bat -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim
wtest.bat -f unique/arbitrator/dn3_mn1_replica_change.sim
#wtest.bat -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim
# lower the priority while file corruption
#wtest.bat -f unique/arbitrator/dn3_mn1_vnode_change.sim
#wtest.bat -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim
#wtest.bat -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim
#wtest.bat -f unique/arbitrator/dn3_mn1_vnode_createErrData_online.sim
wtest.bat -f unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim
wtest.bat -f unique/arbitrator/dn3_mn1_vnode_delDir.sim
wtest.bat -f unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim
wtest.bat -f unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim
wtest.bat -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim
wtest.bat -f unique/arbitrator/dn3_mn2_killDnode.sim
wtest.bat -f unique/arbitrator/insert_duplicationTs.sim
wtest.bat -f unique/arbitrator/offline_replica2_alterTable_online.sim
wtest.bat -f unique/arbitrator/offline_replica2_alterTag_online.sim
wtest.bat -f unique/arbitrator/offline_replica2_createTable_online.sim
wtest.bat -f unique/arbitrator/offline_replica2_dropDb_online.sim
wtest.bat -f unique/arbitrator/offline_replica2_dropTable_online.sim
wtest.bat -f unique/arbitrator/offline_replica3_alterTable_online.sim
wtest.bat -f unique/arbitrator/offline_replica3_alterTag_online.sim
wtest.bat -f unique/arbitrator/offline_replica3_createTable_online.sim
wtest.bat -f unique/arbitrator/offline_replica3_dropDb_online.sim
wtest.bat -f unique/arbitrator/offline_replica3_dropTable_online.sim
wtest.bat -f unique/arbitrator/replica_changeWithArbitrator.sim
wtest.bat -f unique/arbitrator/sync_replica2_alterTable_add.sim
wtest.bat -f unique/arbitrator/sync_replica2_alterTable_drop.sim
wtest.bat -f unique/arbitrator/sync_replica2_dropDb.sim
wtest.bat -f unique/arbitrator/sync_replica2_dropTable.sim
wtest.bat -f unique/arbitrator/sync_replica3_alterTable_add.sim
wtest.bat -f unique/arbitrator/sync_replica3_alterTable_drop.sim
wtest.bat -f unique/arbitrator/sync_replica3_dropDb.sim
wtest.bat -f unique/arbitrator/sync_replica3_dropTable.sim
wtest.bat -f unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim
wtest.bat -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim
wtest.bat -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim
wtest.bat -f unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim
wtest.bat -f general/connection/test_old_data.sim
wtest.bat -f unique/dnode/datatrans_3node.sim
wtest.bat -f unique/dnode/datatrans_3node_2.sim

21
tests/script/sh/deploy.sh
View File

@ -118,49 +118,30 @@ fi
echo " " >> $TAOS_CFG
echo "firstEp ${HOSTNAME}:7100" >> $TAOS_CFG
echo "secondEp ${HOSTNAME}:7200" >> $TAOS_CFG
echo "fqdn ${HOSTNAME}" >> $TAOS_CFG
echo "serverPort ${NODE}" >> $TAOS_CFG
echo "dataDir $DATA_DIR" >> $TAOS_CFG
echo "logDir $LOG_DIR" >> $TAOS_CFG
echo "debugFlag 0" >> $TAOS_CFG
echo "mDebugFlag 143" >> $TAOS_CFG
echo "sdbDebugFlag 143" >> $TAOS_CFG
echo "dDebugFlag 143" >> $TAOS_CFG
echo "vDebugFlag 143" >> $TAOS_CFG
echo "tsdbDebugFlag 143" >> $TAOS_CFG
echo "cDebugFlag 143" >> $TAOS_CFG
echo "jnidebugFlag 143" >> $TAOS_CFG
echo "odbcdebugFlag 143" >> $TAOS_CFG
echo "httpDebugFlag 143" >> $TAOS_CFG
echo "monDebugFlag 143" >> $TAOS_CFG
echo "mqttDebugFlag 143" >> $TAOS_CFG
echo "qdebugFlag 143" >> $TAOS_CFG
echo "rpcDebugFlag 143" >> $TAOS_CFG
echo "tmrDebugFlag 131" >> $TAOS_CFG
echo "udebugFlag 143" >> $TAOS_CFG
echo "sdebugFlag 143" >> $TAOS_CFG
echo "wdebugFlag 143" >> $TAOS_CFG
echo "cqdebugFlag 143" >> $TAOS_CFG
echo "monitor 0" >> $TAOS_CFG
echo "monitorInterval 1" >> $TAOS_CFG
echo "http 0" >> $TAOS_CFG
echo "slaveQuery 0" >> $TAOS_CFG
echo "numOfThreadsPerCore 2.0" >> $TAOS_CFG
echo "defaultPass taosdata" >> $TAOS_CFG
echo "numOfLogLines 20000000" >> $TAOS_CFG
echo "mnodeEqualVnodeNum 0" >> $TAOS_CFG
echo "balanceInterval 1" >> $TAOS_CFG
echo "clog 2" >> $TAOS_CFG
#echo "cache 1" >> $TAOS_CFG
echo "days 10" >> $TAOS_CFG
echo "statusInterval 1" >> $TAOS_CFG
echo "maxVgroupsPerDb 4" >> $TAOS_CFG
echo "minTablesPerVnode 4" >> $TAOS_CFG
echo "maxTablesPerVnode 1000" >> $TAOS_CFG
echo "tableIncStepPerVnode 10000" >> $TAOS_CFG
echo "asyncLog 0" >> $TAOS_CFG
echo "numOfMnodes 1" >> $TAOS_CFG
echo "locale en_US.UTF-8" >> $TAOS_CFG
echo "fsync 0" >> $TAOS_CFG
echo "telemetryReporting 0" >> $TAOS_CFG
echo " " >> $TAOS_CFG

41
tests/script/tmp/182.sim
View File

@ -1,41 +0,0 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3
system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/cfg.sh -n dnode2 -c walLevel -v 2
system sh/cfg.sh -n dnode3 -c walLevel -v 2
system sh/cfg.sh -n dnode4 -c walLevel -v 2
system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode1 -c activeCode -v eglxDLzRpslJWl7OxrPZ2K3sQ5631AP9SVpezsaz2dhJWl7OxrPZ2ElaXs7Gs9nYSVpezsaz2djGIj5StnQ3ZvLHcsE8cwcN
system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 100000
system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 100000
system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 100000
system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 100000
system sh/cfg.sh -n dnode1 -c http -v 1
system sh/cfg.sh -n dnode2 -c http -v 1
system sh/cfg.sh -n dnode3 -c http -v 1
system sh/cfg.sh -n dnode4 -c http -v 1
system sh/cfg.sh -n dnode1 -c httpMaxThreads -v 4
system sh/cfg.sh -n dnode1 -c firstEp -v 127.0.0.1:6030
system sh/cfg.sh -n dnode1 -c secondEp -v 127.0.0.1:6030
system sh/cfg.sh -n dnode1 -c serverPort -v 6030
system sh/cfg.sh -n dnode1 -c fqdn -v 127.0.0.1
system sh/exec.sh -n dnode1 -s start

78
tests/script/tmp/dnodes.sim Normal file
View File

@ -0,0 +1,78 @@
system sh/stop_dnodes.sh
############## config parameter #####################
$node1 = 192.168.101.174
$node2 = 192.168.0.202
$node2 = 192.168.0.203
$node3 = 192.168.0.204
$first = 1
$num = 5
$self = $node1
############### deploy firstEp #####################
$firstEp = $node1 . :7100
$firstPort = 7100
if $first == 1 then
system sh/deploy.sh -n dnode1 -i 1
system sh/cfg.sh -n dnode1 -c firstEp -v $firstEp
system sh/cfg.sh -n dnode1 -c secondEp -v $firstEp
system sh/cfg.sh -n dnode1 -c fqdn -v $node1
system sh/cfg.sh -n dnode1 -c serverPort -v $firstPort
system sh/exec.sh -n dnode1 -s start
sql connect
$i = 0
while $i < $num
$port = $i * 100
$port = $port + 8000
$i = $i + 1
sql create dnode $node1 port $port
endw
$i = 0
while $i < $num
$port = $i * 100
$port = $port + 8000
$i = $i + 1
sql create dnode $node2 port $port
endw
$i = 0
while $i < $num
$port = $i * 100
$port = $port + 8000
$i = $i + 1
sql create dnode $node3 port $port
endw
$i = 0
while $i < $num
$port = $i * 100
$port = $port + 8000
$i = $i + 1
sql create dnode $node4 port $port
endw
endi
############### deploy nodes #####################
$i = 0
while $i < $num
$index = $i + 80
$port = $i * 100
$port = $port + 8000
$dnodename = dnode . $index
$i = $i + 1
system sh/deploy.sh -n $dnodename -i 1
system sh/cfg.sh -n $dnodename -c firstEp -v $firstEp
system sh/cfg.sh -n $dnodename -c secondEp -v $firstEp
system sh/cfg.sh -n $dnodename -c fqdn -v $self
system sh/cfg.sh -n $dnodename -c serverPort -v $port
#system sh/exec.sh -n $dnodename -s start
endw

35
tests/script/tmp/http.sim
View File

@ -1,35 +0,0 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3
system sh/deploy.sh -n dnode4 -i 4
system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/cfg.sh -n dnode2 -c walLevel -v 2
system sh/cfg.sh -n dnode3 -c walLevel -v 2
system sh/cfg.sh -n dnode4 -c walLevel -v 2
system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode1 -c http -v 1
system sh/cfg.sh -n dnode2 -c http -v 1
system sh/cfg.sh -n dnode3 -c http -v 1
system sh/cfg.sh -n dnode1 -c monitor -v 1
system sh/cfg.sh -n dnode2 -c monitor -v 1
system sh/cfg.sh -n dnode3 -c monitor -v 1
system sh/cfg.sh -n dnode1 -c monitorInterval -v 1
system sh/cfg.sh -n dnode2 -c monitorInterval -v 1
system sh/cfg.sh -n dnode3 -c monitorInterval -v 1
system sh/exec.sh -n dnode1 -s start

115
tests/script/tmp/mnodes.sim
View File

@ -1,115 +0,0 @@
system sh/stop_dnodes.sh
system sh/deploy.sh -n dnode1 -i 1
system sh/deploy.sh -n dnode2 -i 2
system sh/deploy.sh -n dnode3 -i 3
system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3
system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3
system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3
system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode1 -c http -v 0
system sh/cfg.sh -n dnode2 -c http -v 0
system sh/cfg.sh -n dnode3 -c http -v 0
system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20000
system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 20000
system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 20000
system sh/cfg.sh -n dnode1 -c minTablesPerVnode -v 1000
system sh/cfg.sh -n dnode2 -c minTablesPerVnode -v 1000
system sh/cfg.sh -n dnode3 -c minTablesPerVnode -v 1000
system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 20
system sh/cfg.sh -n dnode2 -c maxVgroupsPerDb -v 20
system sh/cfg.sh -n dnode3 -c maxVgroupsPerDb -v 20
system sh/cfg.sh -n dnode1 -c replica -v 3
system sh/cfg.sh -n dnode2 -c replica -v 3
system sh/cfg.sh -n dnode3 -c replica -v 3
print ============== deploy
system sh/exec.sh -n dnode1 -s start
sql connect
sql create dnode $hostname2
sql create dnode $hostname3
system sh/exec.sh -n dnode2 -s start
system sh/exec.sh -n dnode3 -s start
print =============== step1
$x = 0
step1:
$x = $x + 1
sleep 1000
if $x == 10 then
return -1
endi
sql show dnodes
print dnode1 $data4_1
print dnode2 $data4_2
print dnode3 $data4_3
if $data4_1 != ready then
goto step1
endi
if $data4_2 != ready then
goto step1
endi
if $data4_3 != ready then
goto step1
endi
sql show mnodes
$mnode1Role = $data2_1
print mnode1Role $mnode1Role
$mnode2Role = $data2_2
print mnode2Role $mnode2Role
$mnode3Role = $data2_3
print mnode3Role $mnode3Role
if $mnode1Role != master then
goto step1
endi
if $mnode2Role != slave then
goto step1
endi
if $mnode3Role != slave then
goto step1
endi
$x = 1
show2:
print =============== step $x
sql show mnodes
print $data0_1 $data2_1
sql show dnodes
print dnode1 $data4_1
print dnode2 $data4_2
print dnode3 $data4_3
if $data4_1 != ready then
goto step1
endi
if $data4_2 != ready then
goto step1
endi
if $data4_3 != ready then
goto step1
endi
$x = $x + 1
sleep 3000
if $x == 100000 then
return -1
endi
goto show2

38
tests/script/tmp/prepare.sim
View File

@ -9,41 +9,3 @@ system sh/cfg.sh -n dnode1 -c walLevel -v 2
system sh/cfg.sh -n dnode2 -c walLevel -v 2
system sh/cfg.sh -n dnode3 -c walLevel -v 2
system sh/cfg.sh -n dnode4 -c walLevel -v 2
system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
system sh/cfg.sh -n dnode1 -c activeCode -v eglxDLzRpslJWl7OxrPZ2K3sQ5631AP9SVpezsaz2dhJWl7OxrPZ2ElaXs7Gs9nYSVpezsaz2djGIj5StnQ3ZvLHcsE8cwcN
system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 100000
system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 100000
system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 100000
system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 100000
system sh/cfg.sh -n dnode1 -c http -v 1
system sh/cfg.sh -n dnode2 -c http -v 1
system sh/cfg.sh -n dnode3 -c http -v 1
system sh/cfg.sh -n dnode4 -c http -v 1
return
# for crash_gen
system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 10
system sh/cfg.sh -n dnode1 -c rpcMaxTime -v 101
system sh/cfg.sh -n dnode1 -c cache -v 2
system sh/cfg.sh -n dnode1 -c keep -v 36500
system sh/cfg.sh -n dnode1 -c walLevel -v 1
# for windows
system sh/cfg.sh -n dnode1 -c firstEp -v 152.136.17.116:6030
system sh/cfg.sh -n dnode1 -c secondEp -v 152.136.17.116:6030
system sh/cfg.sh -n dnode1 -c serverPort -v 6030
system sh/cfg.sh -n dnode1 -c fqdn -v 152.136.17.116

33
tests/script/unique/dnode/basic1.sim
View File

@ -33,7 +33,9 @@ if $data02 != master then
endi
print =============== create dnodes
sql create dnode $hostname2
sql create dnode $hostname port 7200
sleep 2000
sql show dnodes;
if $rows != 2 then
return -1
@ -47,6 +49,22 @@ if $data10 != 2 then
return -1
endi
if $data02 != 0 then
return -1
endi
if $data12 != 0 then
return -1
endi
if $data04 != ready then
return -1
endi
if $data14 != ready then
return -1
endi
sql show mnodes;
if $rows != 1 then
return -1
@ -56,7 +74,7 @@ if $data00 != 1 then
return -1
endi
if $data01 != master then
if $data02 != master then
return -1
endi
@ -76,5 +94,16 @@ if $rows != 2 then
return -1
endi
print =============== drop dnode
sql drop dnode 2;
sql show dnodes;
if $rows != 1 then
return -1
endi
if $data00 != 1 then
return -1
endi
system sh/exec.sh -n dnode1 -s stop -x SIGINT
system sh/exec.sh -n dnode2 -s stop -x SIGINT

36
tests/tsim/src/simSystem.c
View File

@ -42,41 +42,7 @@ char *simParseArbitratorName(char *varName) {
char *simParseHostName(char *varName) {
static char hostName[140];
int32_t index = atoi(varName + 8);
int32_t port = 7100;
switch (index) {
case 1:
port = 7100;
break;
case 2:
port = 7200;
break;
case 3:
port = 7300;
break;
case 4:
port = 7400;
break;
case 5:
port = 7500;
break;
case 6:
port = 7600;
break;
case 7:
port = 7700;
break;
case 8:
port = 7800;
break;
case 9:
port = 7900;
break;
}
sprintf(hostName, "'%s:%d'", simHostName, port);
// simInfo("hostName:%s", hostName);
sprintf(hostName, "%s", simHostName);
return hostName;
}